1 Commits
Author SHA1 Message Date
jschoubben cf6fcdff3c Genesis reads the registry's and the builder's manifests from the catalogue, not the control plane's
The control plane's manifest comes out of the build the installer runs (novox/hq ADR
0069, 04-ISSUES/072); the catalogue no longer holds a copy, and a bed that demanded
one would stop a genesis that is about to succeed.
2026-09-21 15:19:25 +02:00
57 changed files with 3176 additions and 2988 deletions
-1
View File
@@ -1,2 +1 @@
node_modules/ node_modules/
node_modules
+1 -9
View File
@@ -165,15 +165,7 @@ export MESH_LAB_ROUTE_PROXY=<somewhere>/route-proxy
`MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what `MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
`suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and `suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built as a claimed; leave it out and it is neither.
repository: a bed installs a catalogue module by reading its manifest from that checkout when it
runs, so the receipt names the catalogue's commit too, and a run taken before a manifest changed
says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
`--no-build` skips this too, and then the bed runs whatever image the store holds.
Check before running a long suite — it says which of these are missing rather than skipping Check before running a long suite — it says which of these are missing rather than skipping
quietly: quietly:
-56
View File
@@ -1,56 +0,0 @@
# A MACHINE IN USE, ADOPTED — and then converged, and returned (novox/hq ADR 0100, ADR 0101).
#
# The mesh replaces a predecessor that is running on the same machines. The anchor here is
# prepared the way the predecessor leaves one: its own firewall (ufw) allowing a served port and
# denying the rest, a service container on that port under a name a catalogue module also uses, a
# file at a path that module declares, a stand-in for the predecessor's configuration sync that
# rewrites the file, and a container holding the registry's port. The bed then raises the mesh on
# it, adopted, and walks the migration the record decides.
#
# hosting (public)
# anchor 192.0.2.10 the machine in use: the predecessor, then the mesh adopted on it
# joiner 192.0.2.20 a fresh machine: the "second machine" that reaches the service and
# enrols through the found firewall; also where a converged genesis on a
# FRESH machine is asked (ADR 0101)
# outsider 192.0.2.30 never enrolled, never on the private network: the probe from outside,
# and the lab's forge — the bed serves the checkouts under test to the
# anchor's builder from here, so nothing on the workstation listens
#
# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the
# bed; `inbound: deny` would load the lab's own table beside it and make that the thing under test.
scenario: adoption
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
# Sized like the one-node bed's anchor: genesis builds the control plane, the base and the
# catalogue's modules here, beside the predecessor's two containers.
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 12GiB
cpus: 6
disk: 60GiB
joiner:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
disk: 20GiB
outsider:
at: { segment: hosting, address: [192.0.2.30] }
egress: true
inbound: allow
memory: 2GiB
cpus: 2
disk: 15GiB
place:
all: [host, runtime]
+30
View File
@@ -0,0 +1,30 @@
# One machine that becomes a mesh and grants a consumer an S3 bucket from an assigned minio provider.
#
# The postgres bed proves the provider/consumer contract for a database; this proves it for object
# storage (novox/hq ADR 0052/0053), on a provider whose code drives the `mc` CLI (so the runtime image
# carries it): minio is assigned, a consumer that requires s3-bucket is assigned, and the mesh mints
# one secret key; minio's provisioner creates a bucket and a service account under the access key the
# mesh derived with the secret it minted, and the consumer reaches its bucket with only that.
scenario: minio-node
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- mesh-controller:development
# minio's runtime, built by scripts/build-module-runtime.sh minio (it carries mc), loaded onto
# the machine.
- mesh-runtime-minio:development
place:
all: [host, runtime]
+30
View File
@@ -0,0 +1,30 @@
# One machine that becomes a mesh and then assigns itself plex's tool runtime.
#
# The audit-node bed proved an assigned *consumer* (novox/hq ADR 0048). This proves an assigned
# module that *serves tools* (ADR 0052): the same first-node foundation, plus plex's tool runtime on
# top. The node enrols itself, the mesh issues plex a broker account scoped to serve.plex.* and
# assigns it, the host runs the runtime container, and a caller invokes plex.plex_reachable over the
# mesh — proof the module runs its own code as its own process under its own scoped account.
scenario: plex-node
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- mesh-controller:development
# Plex's tool runtime, built by scripts/build-module-runtime.sh plex into the local daemon and
# loaded onto the machine, which holds it by its own image ID.
- mesh-runtime-plex:development
place:
all: [host, runtime]
+29
View File
@@ -0,0 +1,29 @@
# One machine that becomes a mesh and grants a consumer a database from an assigned postgres provider.
#
# The redis mesh-grant bed proves the whole provider/consumer contract for a cache; this proves it for
# a database (novox/hq ADR 0052/0053): postgres's runtime carries psql, its provisioner creates a role
# and database under the login the mesh derived with the password the mesh minted, and a consumer
# connects to its own database with only what the mesh delivered.
scenario: postgres-node
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- mesh-controller:development
# postgres's runtime, built by scripts/build-module-runtime.sh postgres (it carries psql), loaded
# onto the machine.
- mesh-runtime-postgres:development
place:
all: [host, runtime]
+1 -4
View File
@@ -1,6 +1,6 @@
# One machine that becomes a mesh and then assigns itself redis — a *provider* module. # One machine that becomes a mesh and then assigns itself redis — a *provider* module.
# #
# A bed proving an assigned module that serves tools (novox/hq ADR 0052) once lived beside this; this proves the same # plex-node proves an assigned module that serves tools (novox/hq ADR 0052). This proves the same
# for a provider: redis's runtime runs its provisioner AND its tools as one process under one scoped # for a provider: redis's runtime runs its provisioner AND its tools as one process under one scoped
# broker account. The provisioner emitting a lifecycle event is the thing 0052 fixes — before it, # broker account. The provisioner emitting a lifecycle event is the thing 0052 fixes — before it,
# the provisioner ran in a container with no broker and its emit could not fire. # the provisioner ran in a container with no broker and its emit could not fire.
@@ -24,9 +24,6 @@ images:
# Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local # Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local
# daemon and loaded onto the machine, which holds it by its own image ID. # daemon and loaded onto the machine, which holds it by its own image ID.
- mesh-runtime-redis:development - mesh-runtime-redis:development
# The vault's, for the beds that install the catalogue's redis: its own password is a secret the
# vault provides (novox/hq ADR 0085).
- mesh-runtime-mesh-vault:development
place: place:
all: [host, runtime] all: [host, runtime]
+27
View File
@@ -0,0 +1,27 @@
# One machine that becomes a mesh and assigns itself sonarr's tool runtime.
#
# plex-node proved a tools+events module that self-detects its token from a mounted config dir; this
# proves the same self-configuring pattern generalises to the Servarr family (novox/hq ADR 0052):
# sonarr's runtime detects its API key from the server's config.xml and serves sonarr's tools over a
# mesh-issued scoped account, with no live Sonarr to reach.
scenario: sonarr-node
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- mesh-controller:development
- mesh-runtime-sonarr:development
place:
all: [host, runtime]
-32
View File
@@ -1,32 +0,0 @@
# The control plane's store restarting while a machine joins (novox/hq issue 083).
#
# Adopting the foundation's store — the first thing a control-node does, and the first thing a
# migration does — recreates it, and for those seconds the control plane cannot write. A machine
# enrolling then used to be refused, or worse, left with its token spent and no identity. This
# raises the foundation on `anchor`, takes its store away, has `laptop` enrol into the gap and
# brings the store back: the enrolment must complete on its own.
scenario: store-window
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
laptop:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 1GiB
images:
- mesh-controller:development
place:
all: [host, runtime]
-40
View File
@@ -1,40 +0,0 @@
# One machine that becomes a mesh, runs the mesh's own certificate authority, and is then given the
# module that makes it trust it — the bed for novox/hq ADR 0147 and issue 129.
#
# The question is narrow and the bed is shaped to answer only it: does a machine holding `ca-trust`
# verify a certificate from the mesh's own authority with no bundle argument and no `-k`, and does
# it stop verifying it when the module is taken away? The authority itself is what is dialled —
# step-ca serves its own API with a leaf it issued — so nothing else has to be right for the answer
# to mean something. No proxy, no routed name, no public issuance: those are the certificates and
# route-forwarding beds, and a trust bed that leaned on them would pass for their reasons.
#
# The negative half is not optional. It is asserted BEFORE the module is assigned and again AFTER it
# is unassigned, because an anchor bed that only ever checks the success is one that would pass on a
# machine that already trusted everything.
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
# MESH_LAB_CATALOG=.../mesh-catalog/modules
# step-ca's image is upstream and pinned by the catalogue; the machine pulls it over its uplink.
# ca-trust carries no image at all — a script, a unit, and the machine's own systemd.
scenario: trust-anchor
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- mesh-controller:development
place:
# Only the host. The authority's image comes from the internet over the machine's uplink, and the
# trust module has nothing to place.
all: [host]
+7 -5
View File
@@ -3,11 +3,11 @@
# The app-postgres provider and the mesh's own foundation store both want host port 5432, so they # The app-postgres provider and the mesh's own foundation store both want host port 5432, so they
# cannot share a machine — the collision that blocked this chain single-node. Here the foundation # cannot share a machine — the collision that blocked this chain single-node. Here the foundation
# (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain — # (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain —
# the baserow and letta CONSUMERS of the one store (baserow keeps its cache inside its own container, # postgres and redis PROVIDERS plus the baserow and letta CONSUMERS that require them. Both
# novox/hq 081). Both
# machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor, # machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor,
# over the underlay both machines already share. The consumers' databases are minted on the one # over the underlay both machines already share. Provider and consumers are co-located on laptop, so
# foundation store on anchor and reached over the overlay; laptop runs no provider of its own. # no cross-node module comms and no overlay are needed — and the 5432-vs-foundation conflict is gone
# because the foundation store is on the OTHER node.
scenario: two-node-db scenario: two-node-db
segments: segments:
@@ -25,7 +25,8 @@ machines:
inbound: allow inbound: allow
memory: 4GiB memory: 4GiB
cpus: 4 cpus: 4
# The DB consumers: the baserow and letta services and their tools runtimes — a handful of # The whole DB-consumer chain: postgres + redis providers, each a server and a broker-bound
# runtime, plus the baserow and letta consumer services and their tools runtimes — a dozen
# containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server). # containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server).
# At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says # At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says
# so — and convergence would present as "the mesh hangs". Six gigabytes gives it room. # so — and convergence would present as "the mesh hangs". Six gigabytes gives it room.
@@ -48,6 +49,7 @@ images:
# provisioner (ADR 0048). # provisioner (ADR 0048).
- mesh-runtime-postgres:development - mesh-runtime-postgres:development
- mesh-runtime-lavinmq:development - mesh-runtime-lavinmq:development
- mesh-runtime-redis:development
- mesh-runtime-baserow:development - mesh-runtime-baserow:development
- mesh-runtime-letta:development - mesh-runtime-letta:development
+1 -1
View File
@@ -3,7 +3,7 @@
# scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set. # scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set.
# #
# Foundation (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the # Foundation (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the
# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres # `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
# providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/ # providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/
# qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR # qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR
# 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push — # 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push —
+1 -10
View File
@@ -33,16 +33,7 @@ SRCS=(); for f in \
pg.d.ts; do pg.d.ts; do
[ -f "$MOD/$f" ] && SRCS+=("$f") [ -f "$MOD/$f" ] && SRCS+=("$f")
done done
# The module compiles against the SDK, which its package.json names and nothing installs: a module TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist >/dev/null )
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
# the types; the image takes the SDK from MESH_SDK below.
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
fi
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
cp -r "$MESH_TOOLS/dist" "$STAGE/dist" cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
+1 -12
View File
@@ -22,18 +22,7 @@ DOCKERFILE="$MESH_CATALOG/modules/route-proxy/Dockerfile"
[ -f "$MESH_CONTROL/examples/route-proxy/main.go" ] || { [ -f "$MESH_CONTROL/examples/route-proxy/main.go" ] || {
echo "no proxy source at $MESH_CONTROL/examples/route-proxy" >&2; exit 1; } echo "no proxy source at $MESH_CONTROL/examples/route-proxy" >&2; exit 1; }
# The bases the manifest declares (novox/hq ADR 0097) are what this build starts FROM — the same
# images the mesh's builder would copy and hand the recipe, not the Dockerfile's floating defaults.
BASES=()
while IFS=$'\t' read -r arg image; do
[ -n "$arg" ] && BASES+=(--build-arg "$arg=$image")
done < <(python3 -c '
import json, sys
for on in json.load(open(sys.argv[1])).get("build", {}).get("on", []):
print(on["arg"], on["image"], sep="\t")
' "$MESH_CATALOG/modules/route-proxy/module.json")
# Context is the mesh-controller repository root: the proxy compiles against that module's go.mod and # Context is the mesh-controller repository root: the proxy compiles against that module's go.mod and
# its examples/route-proxy package. # its examples/route-proxy package.
docker build -f "$DOCKERFILE" "${BASES[@]}" -t "$TAG" "$MESH_CONTROL" docker build -f "$DOCKERFILE" -t "$TAG" "$MESH_CONTROL"
echo "built $TAG (from $MESH_CONTROL/examples/route-proxy)" echo "built $TAG (from $MESH_CONTROL/examples/route-proxy)"
+4 -11
View File
@@ -16,7 +16,6 @@
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import { repositories } from "./repos.ts"; import { repositories } from "./repos.ts";
import { plannedRuntimes } from "./runtimes.ts";
export interface Build { export interface Build {
/** What it produces, for the log. */ /** What it produces, for the log. */
@@ -113,16 +112,10 @@ export function carriedImage(env: NodeJS.ProcessEnv = process.env): string {
return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development"; return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development";
} }
/** /** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
* rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
*
* The plan is what the run was pointed at, plus the module runtimes the named beds stock where
* those are older than their source (novox/hq 04-ISSUES/075) — so a bed never again passes against
* a runtime built two weeks before the manifest it serves.
*/
export function rebuild(env: NodeJS.ProcessEnv = process.env, testFiles: string[] = []): string[] {
const built: string[] = []; const built: string[] = [];
for (const build of [...planned(env), ...plannedRuntimes(testFiles, env)]) { for (const build of planned(env)) {
const [command, ...args] = build.argv; const [command, ...args] = build.argv;
const ran = spawnSync(command!, args, { const ran = spawnSync(command!, args, {
cwd: build.in, cwd: build.in,
@@ -134,7 +127,7 @@ export function rebuild(env: NodeJS.ProcessEnv = process.env, testFiles: string[
// the code in front of you, which is the whole of 005. // the code in front of you, which is the whole of 005.
throw new Error( throw new Error(
`could not build the ${build.what}: ${build.argv.join(" ")} in ${build.in}\n\n` + `could not build the ${build.what}: ${build.argv.join(" ")} in ${build.in}\n\n` +
`${(ran.stderr || ran.stdout || (ran.error ? String(ran.error) : `exit status ${ran.status}, and it said nothing`)).trim()}`, `${(ran.stderr || ran.stdout || String(ran.error)).trim()}`,
); );
} }
built.push(build.what); built.push(build.what);
+1 -12
View File
@@ -10,7 +10,7 @@
* pointed at is neither built nor claimed. * pointed at is neither built nor claimed.
*/ */
import { basename, dirname } from "node:path"; import { dirname } from "node:path";
export interface Repositories { export interface Repositories {
/** Absolute path to the repository root, by name. */ /** Absolute path to the repository root, by name. */
@@ -24,16 +24,5 @@ export function repositories(env: NodeJS.ProcessEnv = process.env): Repositories
if (host) found["mesh-host"] = dirname(host); if (host) found["mesh-host"] = dirname(host);
const modules = env["MESH_LAB_MODULES"]; const modules = env["MESH_LAB_MODULES"];
if (modules) found["mesh-controller"] = dirname(dirname(modules)); if (modules) found["mesh-controller"] = dirname(dirname(modules));
// The catalogue is read, not built: a bed installs a module by reading its manifest from this
// checkout at run time (novox/hq 04-ISSUES/073). A receipt that did not name the catalogue's
// commit could not say whether a catalogue change had been proven — the beds used to carry
// their own copies of the manifests, and then it could not.
const catalogue = env["MESH_LAB_CATALOG"];
if (catalogue) found["mesh-catalog"] = catalogueRoot(catalogue);
return found; return found;
} }
/** MESH_LAB_CATALOG is accepted under either spelling — the checkout, or its `modules` directory. */
export function catalogueRoot(catalogue: string): string {
return basename(catalogue) === "modules" ? dirname(catalogue) : catalogue;
}
-144
View File
@@ -1,144 +0,0 @@
/**
* The module runtimes a run stocks are rebuilt by the run, like everything else it tests.
*
* A per-module bed stocks the module's runtime image — the tool runtime carrying that module's
* code — from the workstation's image store, by tag. It was built by hand, by a script the suite
* never called, and on the day this was written the images for six modules about to run dated
* from two weeks before the manifests they were installed with (novox/hq 04-ISSUES/075). The
* suite's own rule, *the run rebuilds what it tests*, was held for the host and the control plane
* and not for these.
*
* So: for the beds about to run, every `mesh-runtime-<module>:development` their scenarios stock
* is compared against the source it is built from — the module in the catalogue, and the tool
* runtime and SDK it is built on — and rebuilt where the image is older, missing, or the source is
* uncommitted. A rebuild that fails stops the suite, the way a stale host binary would.
*/
import { readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import { parse } from "yaml";
import type { Build } from "./rebuild.ts";
import { catalogueRoot } from "./repos.ts";
/** A runtime image a scenario stocks by tag, and the module it is built from. */
export interface StockedRuntime {
tag: string;
module: string;
}
/**
* Tags whose name is not the module's. The audit logger's runtime was the first, built before the
* script was generalised, and the scenario still stocks it under the module's slug.
*/
const NAMED_OTHERWISE: Record<string, string> = { "mesh-runtime-audit": "audit-logger" };
const RUNTIME_TAG = /^(mesh-runtime-[a-z0-9-]+):development$/;
/** The runtimes the scenarios of these beds stock, each named once. */
export function runtimesStockedBy(testFiles: string[], root: string = process.cwd()): StockedRuntime[] {
const seen = new Map<string, StockedRuntime>();
for (const file of testFiles) {
const text = readFileSync(resolve(root, file), "utf8");
const named = /const SCENARIO = "([^"]+)"/.exec(text);
if (!named) continue;
const scenario = parse(readFileSync(join(root, "scenarios", `${named[1]}.yml`), "utf8")) as { images?: unknown };
for (const image of Array.isArray(scenario.images) ? scenario.images : []) {
const m = RUNTIME_TAG.exec(String(image));
if (!m) continue;
const repository = m[1]!;
seen.set(repository, { tag: String(image), module: NAMED_OTHERWISE[repository] ?? repository.slice("mesh-runtime-".length) });
}
}
return [...seen.values()];
}
/** When an image was made and when its source last changed, in seconds; null for no image. */
export interface Ages {
image: number | null;
/** Infinity where the source has uncommitted changes: what is on disk is newer than any commit. */
source: number;
}
/** stale is whether the image predates its source, or is not there at all. */
export function isStale(a: Ages): boolean {
return a.image === null || a.image < a.source;
}
/** A command runner, for the two questions asked below; injected so the rules can be tested. */
export type Ask = (command: string, args: string[]) => { status: number | null; stdout: string };
const ask: Ask = (command, args) => {
const ran = spawnSync(command, args, { encoding: "utf8" });
return { status: ran.status, stdout: ran.stdout ?? "" };
};
/** ageOfImage is when the local image store made this tag, or null when it holds no such image. */
export function ageOfImage(tag: string, run: Ask = ask): number | null {
const ran = run("docker", ["image", "inspect", "--format", "{{.Created}}", tag]);
if (ran.status !== 0) return null;
const at = Date.parse(ran.stdout.trim());
return Number.isNaN(at) ? null : Math.floor(at / 1000);
}
/**
* ageOfSource is the newest commit touching what the runtime is built from: the module's directory
* in the catalogue, and the whole of each repository it is built on top of. Uncommitted changes in
* any of them are newer than every commit.
*/
export function ageOfSource(catalogue: string, module: string, builtOn: string[], run: Ask = ask): number {
let newest = 0;
const at = (dir: string, path?: string): number => {
const args = ["-C", dir, "log", "-1", "--format=%ct"];
if (path) args.push("--", path);
const ran = run("git", args);
const t = Number.parseInt(ran.stdout.trim(), 10);
return ran.status === 0 && Number.isFinite(t) ? t : 0;
};
const dirty = (dir: string, path?: string): boolean => {
const args = ["-C", dir, "status", "--porcelain"];
if (path) args.push("--", path);
const ran = run("git", args);
return ran.status === 0 && ran.stdout.trim() !== "";
};
if (dirty(catalogue, `modules/${module}`)) return Infinity;
newest = Math.max(newest, at(catalogue, `modules/${module}`));
for (const dir of builtOn) {
if (dirty(dir)) return Infinity;
newest = Math.max(newest, at(dir));
}
return newest;
}
/**
* plannedRuntimes is the runtime builds these beds need before they run, given where the run was
* pointed: nothing where no catalogue was named (the beds skip), one build per stale image
* otherwise. The repositories the runtime is built on are the siblings the build script itself
* reads (`MESH_TOOLS`, `MESH_SDK`, or the checkouts beside this one).
*/
export function plannedRuntimes(testFiles: string[], env: NodeJS.ProcessEnv = process.env,
root: string = process.cwd(), run: Ask = ask): Build[] {
const named = env["MESH_LAB_CATALOG"];
if (!named) return [];
const catalogue = catalogueRoot(named);
const builtOn = [
env["MESH_TOOLS"] ?? resolve(root, "..", "mesh-tools"),
env["MESH_SDK"] ?? resolve(root, "..", "mesh-sdk"),
];
const builds: Build[] = [];
for (const runtime of runtimesStockedBy(testFiles, root)) {
const ages: Ages = {
image: ageOfImage(runtime.tag, run),
source: ageOfSource(catalogue, runtime.module, builtOn, run),
};
if (!isStale(ages)) continue;
builds.push({
what: `runtime ${runtime.tag}`,
in: root,
argv: ["scripts/build-module-runtime.sh", runtime.module, join(tmpdir(), `mesh-lab-${runtime.module}.tar`)],
env: { MESH_CATALOG: catalogue, RUNTIME_TAG: runtime.tag },
});
}
return builds;
}
+1 -1
View File
@@ -40,7 +40,7 @@ export async function runSuite(args: string[]): Promise<number> {
if (!args.includes("--no-build")) { if (!args.includes("--no-build")) {
// Before the run, always. The artifacts are built from two other repositories, and a suite // Before the run, always. The artifacts are built from two other repositories, and a suite
// that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005). // that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
const built = rebuild(process.env, files); const built = rebuild();
if (built.length > 0) console.log(`built: ${built.join(", ")}\n`); if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
} }
// Read now, while it is true. The receipt names these, and reading them when the run ends // Read now, while it is true. The receipt names these, and reading them when the run ends
-96
View File
@@ -1,96 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
/**
* A bed installs a catalogue module by reading the catalogue, never by carrying a copy.
*
* The beds used to build the manifests they install inline, as literals taken from the catalogue
* when each bed was written. The copies did not move when the catalogue did: six modules were
* converted to file-delivered secrets and not one bed ran the converted shape, because every bed
* ran its own copy (novox/hq 04-ISSUES/073). "Proven in the lab" then meant "the copy was proven".
*
* So: a manifest literal in a bed that names a catalogue module is refused, unless the bed is
* listed below with the reason it still carries one. The list is the debt, and it only shrinks.
*
* What this reads: `module: "<name>"` and `"module": "<name>"` with a `version` close by, either
* order, in test/integration/*.test.ts, against the catalogue's directory names. Skipped aloud
* where MESH_LAB_CATALOG is unset — a skip is reported, never silent. A bed that hid the name
* behind a computed string would pass — this is a fence, not a proof, and the reviewer of a bed
* that builds a manifest inline is the proof.
*/
/**
* Beds that still carry an inline copy of a catalogue module's manifest, and why. Three reasons
* recur, and each names the work that removes the entry:
*
* BESIDE the catalogue's module CLAIMS the foundation's container (postgres claims mesh-store,
* lavinmq mesh-broker) and adopts it in place; the bed raises a second one beside the
* foundation's instead. Reading the catalogue changes what the bed raises — it would
* adopt — and the bed's assertions with it.
* WEARING the bed proves a mesh mechanism (a grant, a credential, a restart, a route) with a
* module cut down to the shape the mechanism needs — no upstream server, a secret in the
* environment, a requirement edge removed — and gives it a catalogue name. It is a mesh
* test wearing a catalogue module's name. It cannot simply be renamed: a module's name
* is its tool namespace and its broker scope, so a fixture running the module's runtime
* must carry the module's name (novox/hq ADR 0093). It reads the catalogue and installs
* what the module requires — the vault for a secret, the route module for a route — or
* it runs no real runtime and carries a name of its own.
* DIFFERS a module bed whose copy differs from the catalogue in more than the lab may rewrite
* (an image, a port, an address). Reading the catalogue is the fix and needs a run.
*/
const STILL_CARRIED: Record<string, { modules: string[]; why: string }> = {
"assigned-catalogue-apps.test.ts": { modules: ["postgres", "mongodb", "unifi", "marrytts"],
why: "BESIDE (postgres); DIFFERS (unifi takes its credentials from the environment, mongodb and marrytts drop listens)" },
"assigned-catalogue-media.test.ts": { modules: ["sonarr", "radarr"],
why: "DIFFERS: both drop the route requirement the catalogue declares, and take their API keys from the environment" },
"assigned-catalogue-small.test.ts": { modules: ["postgres", "minio", "redis", "plex"],
why: "BESIDE (postgres); DIFFERS (minio's root password by env-file, redis minting its own secret instead of the vault's, plex without its server)" },
"assigned-model-usage.test.ts": { modules: ["postgres"], why: "BESIDE" },
"assigned-two-node-db.test.ts": { modules: ["baserow", "letta"],
why: "DIFFERS: baserow drops its route requirement, letta drops its ports" },
"lavinmq-bed.test.ts": { modules: ["lavinmq", "amqp-ping"],
why: "BESIDE (lavinmq, with a bootstrap step and a data directory the catalogue has not got); DIFFERS (amqp-ping names its entrypoint)" },
};
const beds = resolve(import.meta.dirname, "integration");
test("a bed that installs a catalogue module reads the catalogue", (t) => {
const absent = catalogueIsPresent();
if (absent) {
// Said, not silent: a check that cannot see the catalogue has checked nothing.
t.skip(`cannot check — ${absent}`);
return;
}
const names = new Set(readdirSync(catalogueDir(), { withFileTypes: true })
.filter((d) => d.isDirectory() && existsSync(resolve(catalogueDir(), d.name, "module.json")))
.map((d) => d.name));
const offences: string[] = [];
for (const file of readdirSync(beds).filter((f) => f.endsWith(".test.ts")).sort()) {
const text = readFileSync(resolve(beds, file), "utf8");
const found = new Set<string>();
// A manifest literal: the module's name with its version close behind it. A `module:` key
// elsewhere (a table of what to register, a grant entry) has no version and is not one.
for (const m of text.matchAll(/(?:^|[\s{,])(?:"module"|module)\s*:\s*"([a-z0-9.-]+)"[^}]{0,160}?(?:"version"|version)\s*:/g)) {
if (names.has(m[1]!)) found.add(m[1]!);
}
// And the other order — a literal that names its version first.
for (const m of text.matchAll(/(?:^|[\s{,])(?:"version"|version)\s*:\s*"[^"]*"[^}]{0,160}?(?:"module"|module)\s*:\s*"([a-z0-9.-]+)"/g)) {
if (names.has(m[1]!)) found.add(m[1]!);
}
const declared = STILL_CARRIED[file];
for (const name of [...found].sort()) {
if (declared?.modules.includes(name)) continue;
offences.push(`${file}: an inline manifest for the catalogue's '${name}'`);
}
for (const name of declared?.modules ?? []) {
if (!found.has(name)) offences.push(`${file}: declared as still carrying '${name}', and it does not — remove the declaration`);
}
}
assert.deepEqual(offences, [],
`a bed carries a copy of a catalogue manifest; read it with catalogueModule() from the harness:\n ${offences.join("\n ")}`);
});
@@ -1,75 +0,0 @@
/**
* **A module that takes a shared-cache grant presents the login it was granted** (novox/hq 081).
*
* The cache provider scopes each consumer to an ACL user of its own, confined to keys under its
* login (novox/hq 080). A consumer that hands its software the password and not the login logs in
* as the server's default user: the grant is honoured by the provider and ignored by the consumer,
* and nothing notices while the default user is open. The grant bed proves the provider's half
* against a consumer written to the contract; this holds every catalogue module that asks for the
* cache to its half — the login, as `${bound:redis-cache:as}`, somewhere it hands its software.
*
* What it cannot see is whether the software also keeps its keys under that login: that is the
* software's, and a module whose software cannot (fixed key or channel names in its code) does not
* take the shared cache at all — baserow runs its own, and n8n in its shipped mode needs none.
*/
import { test } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
const CACHE = "redis-cache";
/** What a module hands its software: every file it writes, and every container's environment and
* arguments. A comment, a `why`, or a declared exception is not handed to anything. */
function handedToSoftware(manifest: string): string[] {
const m = JSON.parse(manifest) as { resources?: Record<string, unknown>[] };
const out: string[] = [];
for (const r of m.resources ?? []) {
if (typeof r["content"] === "string") out.push(r["content"] as string);
if (r["env"] && typeof r["env"] === "object") out.push(...Object.values(r["env"] as Record<string, string>).map(String));
if (Array.isArray(r["args"])) out.push(...(r["args"] as unknown[]).map(String));
}
return out;
}
/** Whether a manifest hands its software the login it is granted for the cache. */
function presentsTheLogin(manifest: string): boolean {
const login = `\${bound:${CACHE}:as}`;
return handedToSoftware(manifest).some((given) => given.includes(login));
}
test("the check sees a module that hands its software the password and not the login", () => {
const passwordOnly = JSON.stringify({ module: "m", requires: [CACHE], resources: [
{ id: "env", type: "file", path: "/x", content: `HOST=\${bound:${CACHE}:at}\nPASSWORD=\${secret:${CACHE}}\n` }] });
const withLogin = JSON.stringify({ module: "m", requires: [CACHE], resources: [
{ id: "env", type: "file", path: "/x", content: `USER=\${bound:${CACHE}:as}\nPASSWORD=\${secret:${CACHE}}\n` }] });
assert.equal(presentsTheLogin(passwordOnly), false);
assert.equal(presentsTheLogin(withLogin), true);
// Named only where no software reads it — a declared reason — is not presenting it.
const onlyInAReason = JSON.stringify({ module: "m", requires: [CACHE], resources: [
{ id: "srv", type: "container", name: "s", image: "x", env: { PASSWORD: `\${secret:${CACHE}}` },
"secrets-in-environment": `the login is \${bound:${CACHE}:as}` }] });
assert.equal(presentsTheLogin(onlyInAReason), false);
});
test("every catalogue module that takes the shared cache presents the login it was granted", (t) => {
const absent = catalogueIsPresent();
if (absent) {
t.skip(`cannot check — ${absent}`);
return;
}
const offences: string[] = [];
for (const d of readdirSync(catalogueDir(), { withFileTypes: true })) {
const file = resolve(catalogueDir(), d.name, "module.json");
if (!d.isDirectory() || !existsSync(file)) continue;
const text = readFileSync(file, "utf8");
const m = JSON.parse(text) as { requires?: string[] };
if (!(m.requires ?? []).includes(CACHE)) continue;
if (!presentsTheLogin(text)) offences.push(d.name);
}
assert.deepEqual(offences, [],
`these take the shared cache and never hand their software the login (\${bound:${CACHE}:as}), so they ` +
`log in as the server's default user — present the login, or run a cache of their own:\n ${offences.join("\n ")}`);
});
-101
View File
@@ -1,101 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { catalogueModule } from "./integration/harness.ts";
import type { HeldImage } from "../src/pinning.ts";
/**
* The shared loader thirteen beds install through (novox/hq 04-ISSUES/073, ADR 0089): it reads
* the catalogue's manifest and rewrites only what the lab must. Checked here against a catalogue
* written by the test, so the rules hold without a lab run.
*/
const digest = (c: string) => "sha256:" + c.repeat(64);
const held: HeldImage[] = [
{ requested: "mesh-runtime-thing:development", repository: "mesh-runtime-thing", reference: digest("a") },
{ requested: "mesh-helper:development", repository: "mesh-helper", reference: digest("b") },
];
function aCatalogueWith(manifest: object): () => void {
const root = mkdtempSync(join(tmpdir(), "mesh-lab-catalogue-"));
mkdirSync(join(root, "modules", "distribution"), { recursive: true });
writeFileSync(join(root, "modules", "distribution", "module.json"), "{}");
mkdirSync(join(root, "modules", "thing"));
writeFileSync(join(root, "modules", "thing", "module.json"), JSON.stringify(manifest));
const before = process.env["MESH_LAB_CATALOG"];
process.env["MESH_LAB_CATALOG"] = root;
return () => {
if (before === undefined) delete process.env["MESH_LAB_CATALOG"]; else process.env["MESH_LAB_CATALOG"] = before;
rmSync(root, { recursive: true, force: true });
};
}
const thing = {
module: "thing", version: "1",
resources: [
{ id: "server", type: "container", name: "thing", image: "postgres@" + digest("c"), ports: ["8080", "9090:9090"], env: { A: "1" } },
{ id: "runtime", type: "container", name: "mesh-thing", artifact: "runtime" },
],
build: { artifacts: [{ name: "runtime", kind: "image", from: "Dockerfile" }] },
};
test("the runtime artifact becomes the image the machine holds, and the build section goes", () => {
const restore = aCatalogueWith(thing);
try {
const m = JSON.parse(catalogueModule("thing", held)) as { build?: unknown; resources: Record<string, unknown>[] };
assert.equal(m.build, undefined);
const runtime = m.resources.find((r) => r["id"] === "runtime")!;
assert.equal(runtime["image"], digest("a"));
assert.equal(runtime["artifact"], undefined);
// An image already pinned to a digest passes through as written.
assert.equal(m.resources.find((r) => r["id"] === "server")!["image"], "postgres@" + digest("c"));
} finally { restore(); }
});
test("an artifact the bed did not name is refused, and a named one resolves to the stocked image", () => {
const helper = { ...thing, resources: [{ id: "helper", type: "container", name: "h", artifact: "helper" }] };
const restore = aCatalogueWith(helper);
try {
assert.throws(() => catalogueModule("thing", held), /names the "helper" artifact/);
const m = JSON.parse(catalogueModule("thing", held, { artifacts: { helper: "mesh-helper" } })) as { resources: Record<string, unknown>[] };
assert.equal(m.resources[0]!["image"], digest("b"));
assert.throws(() => catalogueModule("thing", held, { artifacts: { helper: "mesh-nothing" } }), /stocked no such image/);
} finally { restore(); }
});
test("a host-port remap and a lab address are the only other things that change", () => {
const restore = aCatalogueWith(thing);
try {
const m = JSON.parse(catalogueModule("thing", held, {
ports: { "8080": "8090:8080" },
env: { server: { B: "2" } },
})) as { resources: Record<string, unknown>[] };
const server = m.resources.find((r) => r["id"] === "server")!;
assert.deepEqual(server["ports"], ["8090:8080", "9090:9090"]);
assert.deepEqual(server["env"], { A: "1", B: "2" });
} finally { restore(); }
});
test("a manifest the catalogue does not have is refused by name", () => {
const restore = aCatalogueWith(thing);
try {
assert.throws(() => catalogueModule("nothing", held), /no manifest for nothing/);
} finally { restore(); }
});
test("an upstream artifact resolves to the reference the manifest pins, as the machine pulls it", () => {
const web = {
module: "thing", version: "1",
resources: [{ id: "server", type: "container", name: "web", artifact: "server" }],
build: { artifacts: [{ name: "server", kind: "upstream", from: "alpine@" + digest("e") }] },
};
const restore = aCatalogueWith(web);
try {
const m = JSON.parse(catalogueModule("thing", held)) as { resources: Record<string, unknown>[] };
assert.equal(m.resources[0]!["image"], "alpine@" + digest("e"));
assert.equal(m.resources[0]!["artifact"], undefined);
} finally { restore(); }
});
@@ -21,25 +21,29 @@
*/ */
import { test, before, after } from "node:test"; import { test, before, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync } from "node:fs"; import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts"; import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
const binary = hostBinaryPath(); const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
const skip = !capability.usable ? `lab not usable: ${capability.why}` const skip = !capability.usable ? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
: catalogueIsPresent(); : false;
const SCENARIO = "adopted-store-cross-node"; const SCENARIO = "adopted-store-cross-node";
const NODE = "node2"; const NODE = "node2";
const catalogDir = process.env["MESH_LAB_CATALOG"]
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
let instanceId = ""; let instanceId = "";
let held: HeldImage[] = []; let held: HeldImage[] = [];
@@ -61,12 +65,29 @@ async function must(machine: string, command: string, timeoutMs?: number): Promi
async function mesh(command: string, timeoutMs?: number): Promise<string> { async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
} }
function pinned(reference: string): string { return onTheMachine(reference, held); }
function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); }
/** The catalogue's manifest as the lab runs it (harness), and whether it needs a broker account. */ /** Load a committed module.json with its container images rewritten to the scenario's pinned digests. */
function loadManifest(name: string): { manifest: string; broker: boolean } { function loadManifest(name: string): { manifest: string; broker: boolean } {
const manifest = catalogueModule(name, held); const path = resolve(catalogDir, name, "module.json");
return { manifest, broker: needsBrokerAccount(manifest) }; const m = JSON.parse(readFileSync(path, "utf8")) as {
resources?: { type: string; image?: string; artifact?: string }[];
};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
if (typeof r.image === "string") {
// A placeholder image (mesh-runtime-<m>@0…0) resolves to the stocked digest, as redis does.
r.image = pinned(r.image);
} else if (typeof r.artifact === "string") {
// The bundle-model bed does not build, so resolve a module's runtime ARTIFACT to its stocked
// image directly — postgres/lavinmq name their provisioner by artifact, not a placeholder.
r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`);
delete r.artifact;
}
}
const manifest = JSON.stringify(m);
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
} }
function tokenFrom(said: string): string { function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
@@ -84,6 +105,7 @@ async function install(name: string, node: string): Promise<void> {
before(async () => { before(async () => {
if (skip) return; if (skip) return;
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`) }); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`) });
instanceId = raised.instanceId; instanceId = raised.instanceId;
held = raised.images; held = raised.images;
-976
View File
@@ -1,976 +0,0 @@
/**
* A MACHINE IN USE IS ADOPTED BEFORE IT IS CONVERGED (novox/hq ADR 0100, and ADR 0101 on what
* "in use" ignores).
*
* The mesh replaces a predecessor running on the same machines. This bed prepares a machine the
* way the predecessor leaves one — the record's own words, "How it is checked":
*
* - its firewall (ufw) allowing a served port and denying the rest, incoming and routed, with the
* predecessor's published container ports filtered through it (the ufw-docker arrangement);
* - a service container, `hello-web`, listening on that port under a name the catalogue's
* `hello-web` module also uses, and a file at a path that module declares;
* - a stand-in for the predecessor's control that rewrites that file, stopped by the operator
* before adoption as the record prescribes, and started again later to play one forgotten;
* - a container holding the registry's port.
*
* Then it asks, in the record's order: a converged genesis refuses; an adopted one refuses the held
* registry port and comes up on another; nothing that serves changed; the store is unreachable
* from outside and reachable where it must be; the mesh works through the found firewall, across a
* reload and a reboot; a predecessor still writing is caught; assigning prepares and taking cuts
* over; converging previews, refuses while a found container is held, flips, and returns.
*
* **The module under migration is the catalogue's `hello-web` with its route requirement taken
* off**, read from the catalogue (never a copy): the route needs a proxy module and a public name,
* neither of which is what adoption is about. Its names — the container, the file, the port — are
* the catalogue's, which is the point: they are what the predecessor also uses.
*
* **The forge is in the lab.** Genesis builds the control plane and the catalogue from a
* repository and a commit; this bed serves the checkouts it was pointed at (their HEADs) from the
* `outsider` machine, so the run builds exactly the code under test and nothing on the workstation
* listens for the lab.
*
* Each step is recorded rather than allowed to throw; a step whose dependency failed is not
* attempted, and the report says which.
*
* MESH_LAB_INCUS='sudo -n incus'
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_LAB_MODULES=<mesh-controller>/examples/modules
* MESH_TOOLS=<mesh-tools> MESH_SDK=<mesh-sdk> (default: the checkouts beside this one)
* MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec, push, instanceNameOf } from "../../src/lifecycle/operate.ts";
import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, HOST_PATH } from "../../src/lifecycle/place.ts";
import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts";
import { incus } from "../../src/incus/client.ts";
import { catalogueRoot } from "../../src/repos.ts";
import { ready, returnTo, keep } from "../../src/warm.ts";
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts";
import { genesis, type GenesisOptions } from "./genesis.ts";
const SCENARIO = "adoption";
const CONTROL = "anchor";
const JOINER = "joiner";
const OUTSIDER = "outsider";
const ANCHOR = "192.0.2.10";
const JOINER_ADDRESS = "192.0.2.20";
const FORGE = "192.0.2.30";
/** The port the predecessor serves, and the module that also names its container and file. */
const SERVED = 8080;
const SERVICE = "hello-web";
const SERVICE_FILE = "/var/lib/hello-web/index.html";
const PREDECESSOR_PAGE = "hello from the predecessor\n";
/** The registry's port, which the predecessor holds — and the one the mesh is given instead. */
const HELD_REGISTRY = 5000;
const REGISTRY_PORT = 5100;
const STORE_PORT = 5432;
const BUS_PORT = 5671;
const HUB_PORT = 51820;
const NETWORK_MODULE = "networking";
const FILTER_MODULE = "nftables";
/** Upstream images, pinned as the catalogue pins them (the harness's table). */
const ALPINE = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b";
const REGISTRY_IMAGE = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const installer = bootstrapBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
const modulesDir = process.env["MESH_LAB_MODULES"] ?? "";
const KEEP = !!process.env["MESH_LAB_KEEP"];
const WARM = !!process.env["MESH_LAB_WARM"];
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "adoption-live" : undefined);
/** The checkouts this run builds from, served by the lab's forge. */
const REPOS: Record<string, string> = {
"mesh-controller": modulesDir ? dirname(dirname(modulesDir)) : "",
"mesh-catalog": catalogDir ? catalogueRoot(catalogDir) : "",
"mesh-tools": process.env["MESH_TOOLS"] ?? resolve(process.cwd(), "..", "mesh-tools"),
"mesh-sdk": process.env["MESH_SDK"] ?? resolve(process.cwd(), "..", "mesh-sdk"),
};
const skip =
!capability.usable ? capability.why :
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
!modulesDir ? "MESH_LAB_MODULES is not set, so there is no control-plane checkout to build from" :
Object.entries(REPOS).find(([, p]) => !p || !existsSync(resolve(p, ".git")))
?.map((x) => `no checkout of ${x[0]} at ${x[1]}`)[0] ?? false;
let instanceId = "";
// ---- talking to the machines ------------------------------------------------------------------
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, machine, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(machine, command, timeoutMs);
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
return out;
}
/** The control plane, retried across the brief windows in which the mesh recreates it. */
async function meshSays(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const deadline = Date.now() + (timeoutMs ?? 120_000);
for (;;) {
const r = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
if (r.ok) return r;
if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon|is restarting/i.test(r.out) &&
Date.now() < deadline) {
await sleep(2_000);
continue;
}
return r;
}
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
const r = await meshSays(command, timeoutMs);
if (!r.ok) throw new Error(`${CONTROL}: mesh-controller ${command}\n${r.out}`);
return r.out;
}
function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
/** Poll until `probe` returns a value, or fail naming the last thing it saw. */
async function until<T>(what: string, seconds: number, probe: () => Promise<T | null>, last: () => string): Promise<T> {
const deadline = Date.now() + seconds * 1000;
for (;;) {
const got = await probe();
if (got !== null) return got;
if (Date.now() > deadline) throw new Error(`${what} — not within ${seconds}s. Last:\n${last()}`);
await sleep(5_000);
}
}
/** Whether a TCP connection from `machine` to address:port opens within three seconds. */
async function connects(machine: string, address: string, port: number): Promise<boolean> {
return (await on(machine, `timeout 4 bash -c ${quote(`</dev/tcp/${address}/${port}`)}`)).ok;
}
/** Register a module with the control plane from a manifest's text. */
async function registerModule(module: string, manifest: string): Promise<string> {
const local = join(tmpdir(), `mesh-lab-adoption-${process.pid}-${module}.json`);
writeFileSync(local, manifest);
await push(instanceId, CONTROL, local, `/tmp/${module}.json`);
await must(CONTROL, `docker cp /tmp/${module}.json mesh-controller:/${module}.json`);
return mesh(`module add /${module}.json`);
}
async function nodeShow(node: string): Promise<string> {
return mesh(`node show ${node}`);
}
/** The anchor's address on the private network. */
async function meshAddressOf(machine: string): Promise<string> {
const out = await must(machine, `ip -4 -o addr show dev mesh0`);
const found = out.match(/inet (\d+\.\d+\.\d+\.\d+)\//)?.[1];
assert.ok(found, `${machine} has no address on mesh0:\n${out}`);
return found;
}
/** The rules ufw was given, one per line, as `ufw show added` prints them. */
async function ufwAdded(): Promise<string[]> {
const out = await must(CONTROL, `ufw show added`);
return out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
}
function marked(rule: string): boolean {
return /comment 'mesh-host /.test(rule);
}
async function restartMachine(machine: string): Promise<void> {
const name = await instanceNameOf(instanceId, machine);
await incus(["restart", name], 180_000);
await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`));
}
/** Until the anchor reports what it was last sent as applied and current. */
async function settled(node = CONTROL, withinMs = 300_000): Promise<void> {
let last = "";
await until(`${node} reports the declaration it was sent as applied and current`, withinMs / 1000, async () => {
const asked = await meshSays(`status --json`);
last = asked.out;
if (!asked.ok) return null;
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === node);
if (bad) throw new Error(`${node} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === node);
return !state.waiting.some((w) => w.node === node) && word?.outcome === "applied" && word.current ? true : null;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
return null;
}
}, () => last);
}
/** Send a node what it should be, and wait until it says it applied it. */
async function pushAndSettle(node: string): Promise<string> {
const said = await mesh(`push ${node}`, 600_000);
await settled(node);
return said;
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
// ---- the lab's forge ---------------------------------------------------------------------------
/**
* Serve the checkouts under test from the outsider machine, over git's own protocol.
*
* Each checkout's HEAD is pushed into a bare repository as `main` and `lab`, the lot is carried in,
* and a git daemon answers on the outsider's scenario address — which the anchor's builder reaches
* over the hosting segment. Returns the commit each repository is served at.
*/
async function raiseForge(): Promise<Record<string, string>> {
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-forge-"));
const heads: Record<string, string> = {};
try {
for (const [name, checkout] of Object.entries(REPOS)) {
const bare = join(dir, `${name}.git`);
execFileSync("git", ["init", "-q", "--bare", bare]);
execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare,
"HEAD:refs/heads/main", "HEAD:refs/heads/lab"], { stdio: "pipe" });
heads[name] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
}
const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`);
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
await push(instanceId, OUTSIDER, tar, "/tmp/forge.tar");
rmSync(tar, { force: true });
} finally {
rmSync(dir, { recursive: true, force: true });
}
await must(OUTSIDER, `command -v git >/dev/null || pacman -S --noconfirm --needed git`, 600_000);
// Owned by the daemon's user: extracted as the workstation's uid, git refuses to serve a
// repository someone else owns ("dubious ownership"), and the clone fails with no reason given.
await must(OUTSIDER, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE} --pid-file=/run/git-daemon.pid`);
await must(OUTSIDER, `git ls-remote git://${FORGE}/mesh-controller.git lab`);
await until("the lab's forge answers the anchor", 60, async () =>
(await connects(CONTROL, FORGE, 9418)) ? true : null, () => "no connection to 9418");
return heads;
}
const forgeUrl = (repo: string) => `git://${FORGE}/${repo}.git`;
// ---- the predecessor ---------------------------------------------------------------------------
/**
* The ufw-docker arrangement: published container ports pass through ufw's route rules, and
* traffic from private ranges is let through. It is how a ufw machine filters what docker publishes
* at all — without it docker's own rules bypass ufw entirely (novox/hq research 012 measured 52
* forwarding rules on the control-node, one per served port).
*/
const UFW_DOCKER = `
# BEGIN UFW AND DOCKER
*filter
:ufw-user-forward - [0:0]
:ufw-docker-logging-deny - [0:0]
:DOCKER-USER - [0:0]
-A DOCKER-USER -j ufw-user-forward
-A DOCKER-USER -j RETURN -s 10.0.0.0/8
-A DOCKER-USER -j RETURN -s 172.16.0.0/12
-A DOCKER-USER -j RETURN -s 192.168.0.0/16
-A DOCKER-USER -p udp -m udp --sport 53 --dport 1024:65535 -j RETURN
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 192.168.0.0/16
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 10.0.0.0/8
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 172.16.0.0/12
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 192.168.0.0/16
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 10.0.0.0/8
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 172.16.0.0/12
-A DOCKER-USER -j RETURN
-A ufw-docker-logging-deny -j DROP
COMMIT
# END UFW AND DOCKER
`;
/** The stand-in for the predecessor's configuration sync: it rewrites the file every ten seconds. */
const STAND_IN = `[Unit]
Description=Stand-in for the predecessor's configuration sync: rewrites a file a catalogue module declares
[Service]
ExecStart=/bin/sh -c 'while true; do printf "hello from the predecessor, synced %%s\\\\n" "$(date +%%s)" > ${SERVICE_FILE}; sleep 10; done'
`;
/** The page the predecessor's service loop serves — the catalogue module's own loop, verbatim. */
const SERVE_LOOP =
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done";
/** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */
const BEFORE = "/root/predecessor";
/** A predecessor container with no restart policy: a runtime restart would lose it. */
const NO_POLICY = "predecessor-nopolicy";
/** The broker's plaintext port: published on every interface, and the filter admits it from the mesh only. */
const AMQP_PORT = 5672;
async function preparePredecessor(): Promise<string> {
const said: string[] = [];
await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000);
const rules = join(tmpdir(), `mesh-lab-ufw-docker-${process.pid}`);
writeFileSync(rules, UFW_DOCKER);
await push(instanceId, CONTROL, rules, "/tmp/ufw-docker.rules");
await must(CONTROL, [
`grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules || cat /tmp/ufw-docker.rules >> /etc/ufw/after.rules`,
`ufw default deny incoming`,
`ufw default allow outgoing`,
`ufw default deny routed`,
`ufw allow 22/tcp`,
`ufw allow ${SERVED}/tcp`,
`ufw route allow proto tcp from any to any port ${SERVED}`,
`ufw --force enable`,
`systemctl enable ufw`,
].join(" && "));
said.push(` firewall ufw: deny incoming and routed; allow 22 and ${SERVED}; ufw-docker after.rules`);
await must(CONTROL, `mkdir -p /var/lib/hello-web && printf %s ${quote(PREDECESSOR_PAGE)} > ${SERVICE_FILE}`);
await must(CONTROL,
`docker run -d --name ${SERVICE} --restart unless-stopped -p ${SERVED}:${SERVED} ` +
`-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000);
await must(CONTROL,
`docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000);
// A container the predecessor left running with no restart policy: a restart of the runtime
// would not bring it back, which is what ADR 0102 forbids the mesh from causing.
await must(CONTROL, `docker run -d --name ${NO_POLICY} ${ALPINE} sleep infinity`, 600_000);
// And a setting of the machine's own in the runtime's file, beside the registries it ships with.
await must(CONTROL,
`python3 - <<'EOF'
import json
f="/etc/docker/daemon.json"
d=json.load(open(f))
d["log-opts"]={"max-size":"7m"}
json.dump(d,open(f,"w"),indent=2)
EOF
systemctl reload docker`);
said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`);
// The predecessor's control, which the operator stops before adopting (the record's words).
const unit = join(tmpdir(), `mesh-lab-stand-in-${process.pid}`);
writeFileSync(unit, STAND_IN);
await push(instanceId, CONTROL, unit, "/etc/systemd/system/predecessor-sync.service");
await must(CONTROL, `systemctl daemon-reload && systemctl start predecessor-sync && sleep 12 && systemctl stop predecessor-sync`);
said.push(` stand-in predecessor-sync rewrote ${SERVICE_FILE}, then the operator stopped it`);
// What the machine was, recorded ON the machine so a restored warm instance still has it.
await must(CONTROL, [
`mkdir -p ${BEFORE}`,
`sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`,
`cp ${SERVICE_FILE} ${BEFORE}/file`,
`docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`,
`docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY} > ${BEFORE}/nopolicy.id`,
`ufw show added > ${BEFORE}/ufw-added.txt`,
].join(" && "));
await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () =>
(await on(JOINER, `curl -s --max-time 3 http://${ANCHOR}:${SERVED}/`)).out.includes("hello from the predecessor") ? true : null,
() => "no answer");
said.push((await must(CONTROL, `ufw status verbose`)).trim());
said.push((await must(CONTROL, `docker ps --format '{{.Names}}\t{{.Ports}}'`)).trim());
return said.join("\n");
}
// ---- steps, recorded rather than thrown --------------------------------------------------------
interface Step { code: string; title: string; ok: boolean; why: string; said: string; seconds: number; warm?: boolean }
const steps = new Map<string, Step>();
async function step(code: string, needs: string[], fn: () => Promise<string>): Promise<void> {
const title = TITLE[code]!;
const missing = needs.filter((n) => !steps.get(n)?.ok);
if (missing.length) {
steps.set(code, { code, title, ok: false, seconds: 0, said: "",
why: `not attempted — ${missing.join(", ")} did not succeed` });
console.log(`[${code}] SKIP ${title}`);
return;
}
console.log(`\n[${code}] ---- ${title} ----`);
const began = Date.now();
const took = () => Math.round((Date.now() - began) / 1000);
try {
const said = await fn();
steps.set(code, { code, title, ok: true, why: "", said, seconds: took() });
console.log(`${said}\n[${code}] PASS ${title} (${took()}s)`);
} catch (err) {
const why = (err as Error).message;
steps.set(code, { code, title, ok: false, why, said: "", seconds: took() });
console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 40).join("\n")}`);
}
}
/** The plan, in the record's order. Codes are stable; titles may be reworded. */
const TITLE: Record<string, string> = {
P0: "the machine is prepared the way the predecessor leaves one",
F0: "a converged genesis on a FRESH machine is not refused — its own resolver does not make it in use (ADR 0101)",
A1: "a converged genesis refuses the machine in use, naming every container and listener it counted",
A2: "an adopted genesis refuses the registry's held port, naming what holds it",
A3: "given another registry port, the adopted foundation comes up — and stays on that port as modules",
B1: "nothing that serves changed: the service answers, its file and container are untouched, the firewall gained only the mesh's marked rules",
B2: "the store is unreachable from outside, before and after the found firewall reloads; the bus answers a machine not yet enrolled",
B4: "the guard lets the machine's own containers reach the store (with the found firewall admitting them)",
C1: "a second machine enrols through the found firewall and joins the private network",
B3: "the store is reachable over the private network",
C2: "after the found firewall reloads, the openings are there and the mesh still works",
C3: "after the machine reboots, the openings are there and the mesh still works",
D1: "assigning prepares: the module holds the found container and file, and neither changes",
D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted",
D3: "converging refuses while the service's module holds its found container",
D4: "taking cuts over: the found container and file are replaced, the original kept, the port reachable",
E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes",
E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed",
E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back",
F1: "unassigning takes the mesh's opening away and leaves the operator's own rule",
};
function stateOutcome(): void {
console.log(`\n================ ADOPTION: WHAT WAS ESTABLISHED ================`);
let established = 0;
for (const code of Object.keys(TITLE)) {
const s = steps.get(code);
const mark = !s ? "NEVER" : s.warm ? "WARM" : s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL";
if (s?.ok) established++;
console.log(` ${code.padEnd(3)} ${mark.padEnd(5)} ${TITLE[code]}${s?.seconds ? ` (${s.seconds}s)` : ""}`);
}
console.log(` ${established}/${Object.keys(TITLE).length} established.`);
}
// ---- the run -----------------------------------------------------------------------------------
before(async () => {
if (skip) return;
console.log(`\n================ THE PLAN ================`);
for (const [code, title] of Object.entries(TITLE)) console.log(` ${code.padEnd(3)} ${title}`);
const verdict = WARM ? await ready(SCENARIO) : { use: "raise" as const, why: "not asked to be warm" };
let restored = false;
if (verdict.use === "restore") {
instanceId = verdict.instanceId;
const seconds = await returnTo(instanceId, (m) => console.log(`warm: ${m}`));
console.log(`WARM: restored ${instanceId} to the adopted foundation in ${seconds}s`);
restored = true;
for (const code of ["P0", "F0", "A1", "A2", "A3"]) {
steps.set(code, { code, title: TITLE[code]!, ok: true, warm: true, seconds: 0, why: "",
said: "restored from the warm snapshot — not re-run; only a fresh run proves it" });
}
} else {
if (WARM) console.log(`WARM: raising — ${verdict.why}`);
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
});
instanceId = bed.instanceId;
}
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
let heads: Record<string, string> = {};
const genesisOn = (node: string, o: Partial<GenesisOptions>): Promise<ReturnType<typeof genesis> extends Promise<infer R> ? R : never> =>
genesis({
instanceId, node, installer: installer as string, catalogDir,
bundleTemplate: foundationBundle(bundle, []),
registry: `${ANCHOR}:${HELD_REGISTRY}`,
source: forgeUrl("mesh-controller"), sourceRef: heads["mesh-controller"] ?? "",
toolsSource: forgeUrl("mesh-tools"), toolsRef: "lab",
catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab",
sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab",
site: "hosting",
// The service, so a machine that reboots comes back holding itself (the bed reboots one).
// F0 asks for a dry run on a machine that must stay fresh, and passes false for itself.
hostService: true,
...(binary ? { hostBinary: binary } : {}),
log: (m) => console.log(m),
...o,
});
if (!restored) {
await step("P0", [], async () => {
heads = await raiseForge();
const said = [` forge git://${FORGE}/ serving ${Object.entries(heads).map(([n, h]) => `${n}@${h.slice(0, 8)}`).join(", ")}`];
said.push(await preparePredecessor());
return said.join("\n");
});
// ADR 0101: the joiner is a freshly installed machine — nothing but its operating system, a
// container runtime and the host binary. A converged genesis there must not be refused. Asked
// as a dry run: the question is the preflight's, and a real raise would make it a second mesh.
await step("F0", ["P0"], async () => {
const ran = await genesisOn(JOINER, { flags: ["--dry-run"], attempts: 1, verify: false, hostService: false });
assert.doesNotMatch(ran.said, /this machine is in use/,
`a converged genesis refused a fresh machine as in use:\n${ran.said}`);
assert.match(ran.said, /in use\s+no: no container runs and nothing listens beyond ssh/,
`the installer never said the fresh machine is not in use:\n${ran.said}`);
const listening = (await must(JOINER, `ss -Hltunp`)).trim();
return ` in use no — the installer went on (${ran.ok ? "dry run finished" : `dry run stopped later, at ${ran.step}`})\n` +
` what listens on the fresh machine:\n${listening.split("\n").map((l) => ` ${l}`).join("\n")}`;
});
await step("A1", ["P0"], async () => {
const ran = await genesisOn(CONTROL, { attempts: 1, verify: false });
assert.ok(!ran.ok, `a converged genesis went ahead on a machine in use:\n${ran.said}`);
assert.match(ran.step, /preflight/, `it was refused, but not before changing anything (at ${ran.step}):\n${ran.said}`);
for (const want of [/container hello-web/, /container predecessor-registry/,
new RegExp(`tcp \\S+:${SERVED} by`), new RegExp(`tcp \\S+:${HELD_REGISTRY} by`)]) {
assert.match(ran.said, want, `the refusal does not name ${want}:\n${ran.said}`);
}
assert.match(ran.said, /--adopted/, `the refusal does not say how to raise it adopted`);
// And nothing was changed: the predecessor as it was, no table of the mesh's.
const ps = await must(CONTROL, `docker ps --format '{{.Names}}'`);
assert.deepEqual(ps.trim().split("\n").sort(), [SERVICE, NO_POLICY, "predecessor-registry"].sort(), `containers changed:\n${ps}`);
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
return ran.said.split("\n").filter((l) => /in use|^\s+- /.test(l)).join("\n");
});
await step("A2", ["A1"], async () => {
const ran = await genesisOn(CONTROL, { adopted: true, attempts: 1, verify: false });
assert.ok(!ran.ok, `an adopted genesis went ahead with the registry's port held:\n${ran.said}`);
assert.match(ran.said, new RegExp(`registry's port tcp/${HELD_REGISTRY} is held by [^\\n]*predecessor-registry`),
`the refusal does not name what holds the registry's port:\n${ran.said.split("\n").slice(-15).join("\n")}`);
assert.match(ran.said, /--registry-port/, `the refusal does not say which flag gives another port`);
const id = (await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim();
assert.equal(id, (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the predecessor's container was replaced`);
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
return ` refused at ${ran.step}\n` + ran.said.split("\n").filter((l) => /held by|port|adopted/.test(l)).slice(-8).join("\n");
});
await step("A3", ["A2"], async () => {
const ran = await genesisOn(CONTROL, { adopted: true, registry: `${ANCHOR}:${REGISTRY_PORT}` });
if (!ran.ok) throw new Error(`${ran.step}: ${ran.why}\n\n${ran.report.join("\n")}`);
await settled();
const said = [ran.report.join("\n")];
const bindings = await must(CONTROL, `docker inspect -f '{{json .HostConfig.PortBindings}}' mesh-registry`);
assert.match(bindings, new RegExp(`"HostPort":"${REGISTRY_PORT}"`), `the registry is not on ${REGISTRY_PORT}: ${bindings}`);
assert.match(await must(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' mesh-registry`), /\S/,
`mesh-registry is not the host's: the foundation was not adopted as a module`);
assert.match(await mesh(`module list`), /^distribution\b/m, `the registry is not a module the mesh holds`);
// The node's own port, in what the mesh would send it — not the catalogue's default.
const plan = await mesh(`plan ${CONTROL} --json`);
assert.match(plan, new RegExp(`"${REGISTRY_PORT}:5000"`), `the registry's module does not publish ${REGISTRY_PORT}`);
assert.doesNotMatch(plan, /"5000:5000"/, `the plan still publishes the catalogue's 5000`);
said.push(` registry on ${REGISTRY_PORT}, as the module the mesh holds: ${bindings.trim()}`);
const show = await nodeShow(CONTROL);
assert.match(show, /mode\s+adopted since/, `the anchor is not reported adopted:\n${show}`);
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `the foundation's dropping table was loaded on an adopted node`);
const guard = await must(CONTROL, `nft list table inet mesh_guard`);
// The guard's port set is the controller's to derive; what the record fixes is that it refuses
// the store's port from outside and holds nothing but refusals.
assert.match(guard, new RegExp(`dport (\\{[^}]*\\b${STORE_PORT}\\b[^}]*\\}|${STORE_PORT}) drop`), `the guard does not refuse the store's port:\n${guard}`);
assert.doesNotMatch(guard, /accept\s*$/m, `the guard holds an accept:\n${guard}`);
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force`);
assert.match(await must(CONTROL, `curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${HELD_REGISTRY}/v2/`), /200/,
`the predecessor's registry stopped answering`);
said.push(show.trim(), guard.trim());
return said.join("\n");
});
if (WARM && steps.get("A3")?.ok) {
await keep(SCENARIO, instanceId);
console.log(`WARM: kept ${instanceId} at the adopted foundation`);
}
}
// ---- nothing that serves changed -------------------------------------------------------------
await step("B1", ["A3"], async () => {
const said: string[] = [];
const want = await must(CONTROL, `cat ${BEFORE}/file`);
let page = "";
await until(`the service answers the joiner as it did`, 120, async () => {
page = (await on(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`)).out;
return page === want ? true : null;
}, () => page);
said.push(` ${SERVICE} answers the joiner on ${SERVED} with the predecessor's page`);
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `${SERVICE_FILE} changed`);
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the ${SERVICE} container was replaced`);
said.push(` file, container byte for byte / the same id as before the mesh`);
const was = (await must(CONTROL, `cat ${BEFORE}/ufw-added.txt`)).split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
const now = await ufwAdded();
const lost = was.filter((r) => !now.includes(r));
const added = now.filter((r) => !was.includes(r));
assert.deepEqual(lost, [], `the found firewall lost rules:\n${lost.join("\n")}`);
const unmarked = added.filter((r) => !marked(r));
assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`);
assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`);
// ADR 0102: the runtime's file is written into, never over, and the runtime is reloaded.
const daemon = JSON.parse(await must(CONTROL, `cat /etc/docker/daemon.json`)) as
{ "log-opts"?: Record<string, string>; "insecure-registries"?: string[] };
assert.equal(daemon["log-opts"]?.["max-size"], "7m", `the machine's own runtime setting was replaced: ${JSON.stringify(daemon)}`);
assert.ok((daemon["insecure-registries"] ?? []).some((r) => r.includes(":")),
`the mesh's registry trust is not in the runtime's file: ${JSON.stringify(daemon)}`);
assert.ok((daemon["insecure-registries"] ?? []).length > 1,
`the machine's own registries were replaced rather than added to: ${JSON.stringify(daemon)}`);
const stillUp = (await must(CONTROL, `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY}`)).trim();
assert.match(stillUp, /^true /, `the container with no restart policy is not running: ${stillUp}`);
assert.equal(stillUp, (await must(CONTROL, `cat ${BEFORE}/nopolicy.id`)).trim(),
`the container with no restart policy was restarted or replaced`);
said.push(` runtime file the machine's log-opts kept, the mesh's registry added; ${NO_POLICY} still up`);
said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`));
return said.join("\n");
});
await step("B2", ["A3"], async () => {
const said: string[] = [];
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers a machine off the private network`);
await must(CONTROL, `ufw reload`);
await sleep(3_000);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the found firewall reloaded`);
said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``);
assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`);
said.push(` outsider -> ${BUS_PORT} the bus answers`);
// **What the guard is for** (ADR 0100, 0103): the store must be unreachable from outside
// *whatever the found firewall does*. With ufw admitting both ports, only the guard is left
// between the outsider and the store — and with the guard gone they are reachable, which is
// what makes this a test of the guard rather than of ufw.
const admit = [STORE_PORT, AMQP_PORT].map((p) =>
`ufw route allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`);
try {
await must(CONTROL, admit.join(" && "));
await sleep(2_000);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)),
`the store answers from outside once the found firewall admits it — the guard refuses nothing`);
assert.ok(!(await connects(OUTSIDER, ANCHOR, AMQP_PORT)),
`the broker's plaintext port answers from outside once the found firewall admits it`);
said.push(` outsider -> ${STORE_PORT}, ${AMQP_PORT} still refused with the found firewall admitting both — the guard's own refusal`);
// The positive control: without the guard, both answer. Anything else would mean the probe
// could not have failed.
await must(CONTROL, `nft delete table inet mesh_guard`);
await sleep(2_000);
const openNow = (await connects(OUTSIDER, ANCHOR, STORE_PORT)) || (await connects(OUTSIDER, ANCHOR, AMQP_PORT));
await must(CONTROL, `systemctl reload mesh-guard.service || systemctl restart mesh-guard.service`);
await sleep(2_000);
assert.ok(openNow, `neither port answered with the guard deleted, so the guard is not what refuses them`);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store stayed open after the guard was loaded again`);
said.push(` guard deleted both answered; loaded again, both refused`);
} finally {
await on(CONTROL, [STORE_PORT, AMQP_PORT].map((p) =>
`ufw route delete allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`).join("; "));
}
return said.join("\n");
});
// Its own step: it asks something different of the found firewall — a container on the machine
// reaches a published port through the runtime's proxy, on the incoming path, not the forwarded.
await step("B4", ["A3"], async () => {
const said: string[] = [];
// What this asks is the guard's promise: it never refuses the machine's own containers. The
// found firewall stays in force (ADR 0100) and denies inbound by default, and a container on
// the store's own network reaches its published port through the runtime's proxy — inbound,
// not forwarded — so the operator's firewall has to admit the container interface for any
// container to get there, on an adopted node as on a converged one. The probe admits it for
// itself alone, and takes the rule away again.
await must(CONTROL, `ufw allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`);
let fromContainer: { ok: boolean; out: string };
try {
fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, 180_000);
} finally {
await on(CONTROL, `ufw delete allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`);
}
if (!fromContainer.ok) {
// Evidence, so the cause can be read from this run rather than guessed at the next one.
const evidence = await on(CONTROL, [
`echo '--- published'; docker ps --format '{{.Names}} {{.Ports}}' | grep -i ${STORE_PORT}`,
`echo '--- from the host'; nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`,
`echo '--- from the host network'; docker run --rm --network host ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`,
`echo '--- iptables FORWARD, DOCKER-USER, isolation'; iptables -S FORWARD; iptables -S DOCKER-USER; iptables -S | grep -i isolation`,
`echo '--- the guard'; nft list table inet mesh_guard`,
`echo '--- nat for the port'; iptables -t nat -S | grep ${STORE_PORT}`,
].join("; "), 120_000);
assert.fail(`a container on the node cannot reach the store:\n${fromContainer.out}\n${evidence.out}`);
}
said.push(` container -> ${STORE_PORT} reachable from a container on the node itself`);
return said.join("\n");
});
// ---- the mesh works through the found firewall -----------------------------------------------
let anchorOnMesh = "";
await step("C1", ["B2"], async () => {
const said: string[] = [];
await mesh(`node add ${JOINER}`);
const token = tokenFrom(await mesh(`token issue --node ${JOINER}`));
const out = await must(JOINER, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
assert.match(out, new RegExp(`enrolled as ${JOINER}`), out);
await must(JOINER, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
said.push(` ${JOINER} enrolled over the bus, through the anchor's ufw`);
await mesh(`overlay place ${JOINER} --site hosting`);
await mesh(`assign ${JOINER} ${NETWORK_MODULE}`);
await pushAndSettle(JOINER);
// The hub's peer list changed: the anchor has to be sent it too.
await pushAndSettle(CONTROL);
anchorOnMesh = await meshAddressOf(CONTROL);
await until(`the joiner reaches the anchor over mesh0`, 180, async () =>
(await on(JOINER, `ping -c1 -W2 ${anchorOnMesh}`)).ok ? true : null,
() => "no ping reply");
said.push(` private network the joiner reaches the anchor at ${anchorOnMesh}`);
said.push((await must(CONTROL, `wg show mesh0 latest-handshakes`)).trim());
return said.join("\n");
});
await step("B3", ["C1"], async () => {
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store does not answer over the private network`);
return ` joiner -> ${anchorOnMesh}:${STORE_PORT} reachable over mesh0`;
});
/** The mesh's own rules in the found firewall. */
const openings = async (): Promise<string[]> => (await ufwAdded()).filter(marked);
const assertOpenings = (rules: string[]) => {
const text = rules.join("\n");
// By the opening's id, which names the machine's port: a forwarded rule names the CONTAINER's
// port (ufw's route rules match after the runtime's translation), so the text may say another.
for (const port of [BUS_PORT, REGISTRY_PORT, HUB_PORT, STORE_PORT]) {
assert.match(text, new RegExp(`opening-(tcp|udp)-${port}-`), `no opening for ${port} among the mesh's rules:\n${text}`);
}
};
await step("C2", ["B3"], async () => {
const before = await openings();
assertOpenings(before);
await must(CONTROL, `ufw reload`);
await sleep(3_000);
const after = await openings();
assert.deepEqual(after.sort(), before.sort(), `the openings changed across a reload`);
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store stopped answering over mesh0 after the reload`);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reload`);
await pushAndSettle(JOINER);
return ` after ufw reload ${after.length} opening(s) in place; the joiner reaches the store over mesh0 and takes a push\n` +
after.map((r) => ` ${r}`).join("\n");
});
await step("C3", ["C2"], async () => {
const before = await openings();
await restartMachine(CONTROL);
await until(`the control plane answers after the reboot`, 300, async () =>
(await meshSays(`status`)).ok ? true : null, () => "no answer");
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force after the reboot`);
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard did not come back`);
const after = await until(`the openings are there again`, 420, async () => {
const now = await openings();
return before.every((r) => now.includes(r)) ? now : null;
}, () => "not all openings");
assertOpenings(after);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reboot`);
await until(`the joiner reaches the store over mesh0 again`, 300, async () =>
(await connects(JOINER, anchorOnMesh, STORE_PORT)) ? true : null, () => "no connection");
await pushAndSettle(JOINER);
const page = await must(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`);
assert.match(page, /hello from the predecessor/, `the predecessor's service did not come back: ${page}`);
return ` after a reboot ufw active, the guard loaded, ${after.length} opening(s); the joiner reaches the store and takes a push`;
});
// ---- assigning prepares, taking cuts over ----------------------------------------------------
let kept = "";
await step("D1", ["B1"], async () => {
const said: string[] = [];
// The catalogue's module, read from the catalogue, with the route requirement taken off: the
// route needs a proxy module and a public name, and neither is what adoption is about.
const manifest = JSON.parse(catalogueModule(SERVICE, [])) as Record<string, unknown>;
delete manifest["requires"]; delete manifest["contributes"]; delete manifest["binds"];
await registerModule(SERVICE, JSON.stringify(manifest));
await mesh(`assign ${CONTROL} ${SERVICE}`);
await pushAndSettle(CONTROL);
const show = await until(`the anchor reports holding ${SERVICE}'s container and file`, 120, async () => {
const s = await nodeShow(CONTROL);
return /holds container\s+hello-web\b/.test(s) && /holds file\s+\/var\/lib\/hello-web\/index\.html/.test(s) ? s : null;
}, () => "");
kept = show.match(/holds file\s+\/var\/lib\/hello-web\/index\.html[^\n]*\n\s*original kept at (\S+)/)?.[1] ?? "";
assert.ok(kept, `the held file's original is not reported kept:\n${show}`);
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `assigning changed ${SERVICE_FILE}`);
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `assigning replaced the ${SERVICE} container`);
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the kept original is not the file as found`);
said.push(show.trim());
return said.join("\n");
});
await step("D2", ["D1"], async () => {
await must(CONTROL, `systemctl start predecessor-sync`);
try {
await sleep(15_000);
await pushAndSettle(CONTROL);
const show = await until(`the anchor reports the held file changed`, 120, async () => {
const s = await nodeShow(CONTROL);
return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null;
}, () => "");
// Stop the writer first, then hash: while it rewrites every ten seconds, a file the host
// reverted in between would still read as the predecessor's a moment later.
await must(CONTROL, `systemctl stop predecessor-sync`);
const was = await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`);
await pushAndSettle(CONTROL);
await sleep(5_000);
const now = await must(CONTROL, `cat ${SERVICE_FILE}`);
assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`);
assert.equal(await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`), was,
`the held file changed under a push after the predecessor stopped writing`);
return show.trim();
} finally {
await on(CONTROL, `systemctl stop predecessor-sync`);
}
});
await step("D3", ["D1"], async () => {
await pushAndSettle(CONTROL);
const r = await meshSays(`converge ${CONTROL}`);
assert.ok(!r.ok, `converge went ahead while ${SERVICE} holds its found container:\n${r.out}`);
assert.match(r.out, new RegExp(`hello-web holds the found container hello-web`), `the refusal does not name the held container:\n${r.out}`);
assert.match(r.out, new RegExp(`take ${CONTROL} hello-web`), `the refusal does not say what to do:\n${r.out}`);
return r.out.trim();
});
await step("D4", ["D1"], async () => {
const said: string[] = [];
said.push((await mesh(`take ${CONTROL} ${SERVICE}`)).trim());
await pushAndSettle(CONTROL);
const label = await until(`the ${SERVICE} container is the mesh's`, 180, async () => {
const l = (await on(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' ${SERVICE}`)).out.trim();
return l && l !== "<no value>" ? l : null;
}, () => "");
assert.notEqual((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the found container was not replaced`);
const catalogue = (JSON.parse(catalogueModule(SERVICE, [])) as { resources: { id: string; content?: string }[] })
.resources.find((r) => r.id === "page")?.content ?? "";
assert.equal(await must(CONTROL, `cat ${SERVICE_FILE}`), catalogue, `the found file was not replaced with the module's`);
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the original was not kept`);
said.push(` replaced container (spec ${label.slice(0, 12)}…) and ${SERVICE_FILE}; original still at ${kept}`);
// Either the mesh opened the port, or the predecessor's own rule already admits it — then the
// mesh adds nothing and will remove nothing (ADR 0103), and the operator's rule stays theirs.
const opened = (await openings()).filter((r) => new RegExp(`opening-tcp-${SERVED}-`).test(r));
const operators = (await ufwAdded()).filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r));
assert.ok(opened.length > 0 || operators.length > 0,
`no opening for ${SERVED} once ${SERVICE} was taken, and no rule of the operator's admits it:\n${(await ufwAdded()).join("\n")}`);
assert.ok(operators.length > 0, `the operator's own rule for ${SERVED} is gone:\n${(await ufwAdded()).join("\n")}`);
said.push(...opened.map((r) => ` opened ${r}`));
if (opened.length === 0) said.push(` opening ${SERVED} satisfied by the operator's own rule: ${operators.join(" | ")}`);
const page = await until(`the taken ${SERVICE} answers over the private network`, 120, async () => {
const p = await on(JOINER, `curl -s --max-time 3 http://${anchorOnMesh}:${SERVED}/`);
return p.ok && p.out === catalogue ? p.out : null;
}, () => "");
said.push(` joiner -> ${SERVED} ${page.trim()}`);
const show = await nodeShow(CONTROL);
assert.doesNotMatch(show, /holds (container|file)\s+\S*hello-web/, `the anchor still holds what was taken:\n${show}`);
return said.join("\n");
});
// ---- converging previews, then changes -------------------------------------------------------
await step("E1", ["D4"], async () => {
if (!/^nftables\b/m.test(await mesh(`module list`))) {
await registerModule(FILTER_MODULE, JSON.stringify(JSON.parse(catalogueModule(FILTER_MODULE, []))));
}
// What the flip will close must be reachable now, or its closing proves nothing.
assert.ok(await connects(JOINER, anchorOnMesh, HELD_REGISTRY),
`the predecessor's published ${HELD_REGISTRY} is not reachable over the private network before the flip`);
await pushAndSettle(CONTROL);
const preview = await mesh(`converge ${CONTROL}`);
assert.match(preview, new RegExp(`tcp/${SERVED}\\b[^\\n]*declared by hello-web`), `the preview does not name the service's port as declared:\n${preview}`);
assert.match(preview, new RegExp(`tcp/${HELD_REGISTRY}\\b[^\\n]*published[^\\n]*WILL CLOSE`), `the preview does not name the published ${HELD_REGISTRY} as closing:\n${preview}`);
assert.match(preview, /the flip takes:\n(\s{4}\S+\n?)+/, `the preview names no module the flip takes:\n${preview}`);
assert.match(preview, new RegExp(`\\n\\s{4}${NETWORK_MODULE}\\b`), `the preview does not say the flip takes ${NETWORK_MODULE}:\n${preview}`);
assert.match(preview, /Nothing has changed/, preview);
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `previewing changed the firewall`);
return preview.trim();
});
await step("E2", ["E1"], async () => {
const said: string[] = [];
// The flip as the preview says to make it — whatever the preview binds `--yes` to.
const preview = await mesh(`converge ${CONTROL}`);
const flip = preview.match(new RegExp(`\`(converge ${CONTROL} --yes[^\`]*)\``))?.[1];
assert.ok(flip, `the preview does not say how to make the flip:\n${preview}`);
said.push((await mesh(flip, 300_000)).trim().split("\n").slice(-3).join("\n"));
await settled();
const table = await until(`the derived filter is loaded`, 300, async () => {
const t = await on(CONTROL, `nft list table inet mesh`);
return t.ok && /policy drop/.test(t.out) ? t.out : null;
}, () => "");
const status = await until(`the found firewall is disabled`, 180, async () => {
const s = (await on(CONTROL, `ufw status`)).out;
return /Status: inactive/.test(s) ? s : null;
}, () => "");
assert.ok((await on(CONTROL, `test -s /etc/ufw/user.rules && grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules`)).ok,
`the found firewall's configuration is not on disk any more`);
assert.match(await nodeShow(CONTROL), /mode\s+converged/, `the anchor is not reported converged`);
said.push(` ufw ${status.trim()} — /etc/ufw/user.rules and after.rules still on disk`);
await until(`the declared ${SERVED} answers over the private network`, 120, async () =>
(await connects(JOINER, anchorOnMesh, SERVED)) ? true : null, () => "");
assert.ok(!(await connects(JOINER, anchorOnMesh, HELD_REGISTRY)), `the undeclared ${HELD_REGISTRY} is still open`);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once converged`);
said.push(` ports ${SERVED} open over the private network; ${HELD_REGISTRY} closed; the store still closed from outside`);
said.push(table.split("\n").slice(0, 30).join("\n"));
return said.join("\n");
});
await step("E3", ["E2"], async () => {
const said = (await mesh(`adopt ${CONTROL}`, 300_000)).trim();
await settled();
await until(`the found firewall is enabled again`, 180, async () =>
/Status: active/.test((await on(CONTROL, `ufw status`)).out) ? true : null, () => "");
await until(`the derived filter is gone`, 180, async () =>
!(await on(CONTROL, `nft list table inet mesh`)).ok ? true : null, () => "");
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard is not restored`);
assertOpenings(await until(`the openings are back`, 180, async () => {
const o = await openings();
return o.length ? o : null;
}, () => ""));
assert.match(await nodeShow(CONTROL), /mode\s+adopted since/, `the anchor is not reported adopted`);
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once adopted again`);
return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`;
});
// ---- what the mesh added, it takes back; what it found, it leaves ---------------------------
await step("F1", ["E3"], async () => {
const said: string[] = [];
const before = await ufwAdded();
const operators = before.filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r));
assert.ok(operators.length > 0, `the operator's own rules for ${SERVED} are already gone:\n${before.join("\n")}`);
await mesh(`unassign ${CONTROL} ${SERVICE}`);
await pushAndSettle(CONTROL);
let lastRules: string[] = before;
const after = await until(`the mesh's own rules for ${SERVED} are gone`, 180, async () => {
const rules = await ufwAdded();
lastRules = rules;
return rules.some((r) => marked(r) && new RegExp(`opening-tcp-${SERVED}-`).test(r)) ? null : rules;
}, () => lastRules.join("\n"));
for (const rule of operators) {
assert.ok(after.includes(rule), `the operator's own rule went with the mesh's opening: ${rule}\n${after.join("\n")}`);
}
said.push(` kept ${operators.length} rule(s) of the operator's, unmarked, after the module was unassigned`);
said.push(...operators.map((r) => ` ${r}`));
return said.join("\n");
});
stateOutcome();
}, { timeout: 10_800_000 });
after(async () => {
if (KEEP || WARM) {
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (${KEEP ? "MESH_LAB_KEEP" : "MESH_LAB_WARM"}).`);
return;
}
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 900_000 });
for (const [code, title] of Object.entries(TITLE)) {
test(`${code} ${title}`, { skip, timeout: 60_000 }, () => {
const s = steps.get(code);
assert.ok(s?.ok, s ? `${s.why}` : `${code} never ran`);
});
}
+59 -11
View File
@@ -49,7 +49,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -62,7 +62,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "anthropic-bed"; const SCENARIO = "anthropic-bed";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -194,6 +194,8 @@ after(async () => {
test("model access refreshes on the manager node and delivers only the access token, never the refresh token", { test("model access refreshes on the manager node and delivers only the access token, never the refresh token", {
skip, timeout: 1_500_000, skip, timeout: 1_500_000,
}, async () => { }, async () => {
const managerImage = pinned("mesh-runtime-anthropic-manager");
const consumerImage = pinned("mesh-runtime-anthropic-consumer");
// --- the licence, and the manager as its holder ------------------------------------------------ // --- the licence, and the manager as its holder ------------------------------------------------
// The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token; // The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token;
@@ -205,15 +207,37 @@ test("model access refreshes on the manager node and delivers only the access to
await mesh(`licence use personal ${MACHINE} anthropic-manager`); await mesh(`licence use personal ${MACHINE} anthropic-manager`);
// --- the manager module, deployed so the host delivers its bound facts -------------------------- // --- the manager module, deployed so the host delivers its bound facts --------------------------
// The catalogue's own manifest (novox/hq 04-ISSUES/073): model-access holder, refresh token bound // Inline manifest mirroring the committed module.json: model-access holder, refresh token bound as a
// as a sealed secret, no node-key mount. The scheduled container installs as present state (ADR // sealed secret, no node-key mount. The scheduled container installs as present state (ADR 0053);
// 0053); the test drives adopt/refresh directly for a deterministic flow rather than waiting on // the test drives adopt/refresh directly for a deterministic flow rather than waiting on cron.
// cron. The one lab rewrite: the OAuth endpoints point at the stub this bed raises below. const managerManifest = JSON.stringify({
const managerManifest = catalogueModule("anthropic-manager", held, { module: "anthropic-manager",
env: { refresh: { version: "1",
requires: ["model-access"],
binds: { "model-access": "/var/lib/mesh/anthropic-manager/model.json" },
secrets: { "model-access": "/var/lib/mesh/anthropic-manager/refresh-token" },
"own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" },
emits: ["module.anthropic-manager.usage.read"],
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" },
{ id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" },
{
id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh",
image: managerImage, network: "host", schedule: "*/9 * * * *",
args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"],
volumes: ["/var/lib/mesh/anthropic-manager:/run/state"],
env: {
MESH_ANTHROPIC_LICENCE: "personal",
MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token", MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token",
MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage", MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage",
} }, MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/refresh-token",
MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token",
MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json",
MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json",
},
},
],
}); });
await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-controller:/anthropic-manager.json`); await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-controller:/anthropic-manager.json`);
await mesh(`module add /anthropic-manager.json`); await mesh(`module add /anthropic-manager.json`);
@@ -305,8 +329,32 @@ test("model access refreshes on the manager node and delivers only the access to
assert.match(submitted, /sealed to 1 holder/, submitted); assert.match(submitted, /sealed to 1 holder/, submitted);
// --- 5. deliver: deploy the consumer and push; it gets the sealed access token ------------------- // --- 5. deliver: deploy the consumer and push; it gets the sealed access token -------------------
// The catalogue's own manifest (novox/hq 04-ISSUES/073). const consumerManifest = JSON.stringify({
const consumerManifest = catalogueModule("anthropic-consumer", held); module: "anthropic-consumer",
version: "1",
requires: ["model-access"],
binds: { "model-access": "/var/lib/anthropic-consumer/model.json" },
secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" },
"own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" },
emits: ["module.anthropic-consumer.usage.session"],
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" },
{ id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" },
{
id: "apply", type: "container", name: "mesh-anthropic-consumer-apply",
image: consumerImage, network: "host", schedule: "*/9 * * * *",
args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"],
volumes: ["/var/lib/anthropic-consumer:/run/state"],
env: {
MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token",
MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json",
MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json",
},
},
],
});
await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-controller:/anthropic-consumer.json`); await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-controller:/anthropic-consumer.json`);
await mesh(`module add /anthropic-consumer.json`); await mesh(`module add /anthropic-consumer.json`);
await mesh(`module issue anthropic-consumer --node ${MACHINE}`); await mesh(`module issue anthropic-consumer --node ${MACHINE}`);
+22 -6
View File
@@ -22,7 +22,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -35,7 +35,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "audit-node"; const SCENARIO = "audit-node";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -145,10 +145,26 @@ after(async () => {
test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", { test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", {
skip, timeout: 900_000, skip, timeout: 900_000,
}, async () => { }, async () => {
// The catalogue's manifest (novox/hq 04-ISSUES/073). Its runtime artifact is the image this // The assigned-module manifest (mesh-catalog), its runtime image the ID the machine holds.
// scenario stocks under the module's slug, `mesh-runtime-audit` — built by scripts/build-runtime-image.sh const manifest = JSON.stringify({
// before build-module-runtime.sh generalised it, and named as it was. module: "audit-logger",
const manifest = catalogueModule("audit-logger", held, { artifacts: { runtime: "mesh-runtime-audit" } }); version: "1",
consumes: ["#"],
"own-secrets": { broker: "/var/lib/audit-logger/broker" },
resources: [
{ id: "state", type: "directory", path: "/var/lib/audit-logger", mode: "0700" },
{ id: "trail", type: "directory", path: "/var/lib/audit-logger/trail", mode: "0700" },
{
id: "run", type: "container", name: "mesh-audit-logger", image: pinned("mesh-runtime-audit"),
network: "host",
volumes: [
"/var/lib/audit-logger/broker:/run/secrets/broker:ro",
"/var/lib/audit-logger/trail:/trail",
],
env: { MESH_BROKER_FILE: "/run/secrets/broker", AUDIT_LOG: "/trail/audit.log" },
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/audit.json && docker cp /tmp/audit.json mesh-controller:/audit.json`); await must(`printf %s ${quote(manifest)} > /tmp/audit.json && docker cp /tmp/audit.json mesh-controller:/audit.json`);
await mesh("module add /audit.json"); await mesh("module add /audit.json");
-221
View File
@@ -1,221 +0,0 @@
/**
* A machine trusts the mesh's own authority because a module put its root there — and stops when
* the module is taken away (novox/hq ADR 0147, 04-ISSUES/129).
*
* What is dialled is the authority itself. step-ca serves its own API with a leaf it issued, so a
* plain client verifying that handshake — no `-k`, no `--cacert` — is verifying exactly one thing:
* that this machine's trust store now contains the mesh's root. No proxy, no routed name, no public
* issuance. A trust bed leaning on those would pass for their reasons, which is the failure this
* whole sequence keeps producing.
*
* The negative half runs twice, before the module is assigned and after it is unassigned. An anchor
* bed that only checks the success would pass on a machine that already trusted everything, and
* would say nothing at all about removal — which is the half issue 129 asked for by name.
*
* MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* MESH_LAB_CATALOG=.../mesh-catalog/modules
* step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink.
* ca-trust carries no image: a script, a unit, and the machine's own systemd.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent();
const SCENARIO = "trust-anchor";
const MACHINE = "anchor";
/** The authority's own API, which it serves with a certificate it issued itself. */
const AUTHORITY = "https://127.0.0.1:9000/health";
/** Where the module puts the root, and therefore what removal must take away. */
const ANCHOR = "/etc/ca-certificates/trust-source/anchors/mesh-internal-ca.crt";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
/** The control plane, a container on the node. */
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
/** Install a catalogue module's manifest into the control plane, read from the catalogue. */
async function register(module: string): Promise<void> {
const manifest = catalogueModule(module, held);
await must(
`printf %s ${quote(manifest)} > /tmp/${module}.json && ` +
`docker cp /tmp/${module}.json mesh-controller:/${module}.json`,
);
await mesh(`module add /${module}.json`);
}
/** Dial the authority the way anything on this machine would: verifying, with nothing handed to it. */
async function verifying(): Promise<{ out: string; ok: boolean }> {
return on(`curl --silent --show-error --max-time 10 ${AUTHORITY}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
// The control plane is a container, and a container that is restarting answers nothing. Asked
// until it answers rather than once, so a crash-looping control plane is reported as itself
// instead of as whatever command happened to be sent first.
let control = { out: "", ok: false };
const answering = Date.now() + 120_000;
while (Date.now() < answering) {
control = await on(`docker exec mesh-controller /mesh-controller status`);
if (control.ok) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(control.ok,
`the control plane never answered:\n${control.out}\n` +
`${(await on(`docker logs mesh-controller 2>&1 | tail -40`)).out}`);
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh's authority is verified on a machine holding ca-trust, and not on one that is not", {
skip, timeout: 1_800_000,
}, async () => {
// The authority first, on its own. Nothing about trust yet.
await register("step-ca");
await mesh(`module issue step-ca --node ${MACHINE}`);
await mesh(`assign ${MACHINE} step-ca`);
await mesh(`push ${MACHINE}`);
await settled();
// It is up and answering — asked the way nothing else in this bed is allowed to ask, with
// verification off, because at this point in the bed no machine could verify it.
let answering = false;
const until = Date.now() + 180_000;
while (Date.now() < until && !answering) {
answering = (await on(`curl --silent --insecure --max-time 5 ${AUTHORITY}`)).ok;
if (!answering) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(answering, `the authority never answered:\n${(await on(`docker logs step-ca 2>&1 | tail -30`)).out}`);
// **The negative half, before anything is assigned.** If this passes, the bed is measuring
// something already in the machine's trust store and everything below it is worthless.
const before = await verifying();
assert.equal(before.ok, false, `the authority verified before anything anchored its root:\n${before.out}`);
assert.match(before.out, /certificate|issuer/i, `it failed for some other reason:\n${before.out}`);
// Now the module.
await register("ca-trust");
await mesh(`assign ${MACHINE} ca-trust`);
await mesh(`push ${MACHINE}`);
await settled();
const unit = await on(`systemctl is-active mesh-ca-trust.service`);
assert.ok(unit.ok, `the unit is ${unit.out.trim()}:\n${(await on(`journalctl -u mesh-ca-trust --no-pager | tail -30`)).out}`);
await must(`test -s ${ANCHOR}`);
const anchors = await must(`trust list | grep -A2 -i 'Mesh Internal CA' || trust list`);
assert.match(anchors, /Mesh Internal CA/, `the mesh's authority is not among the machine's anchors:\n${anchors}`);
// **The whole claim, in one command.** No -k, no --cacert: the machine's own trust store, and a
// certificate the mesh's authority issued.
const after = await verifying();
assert.ok(after.ok, `the authority still does not verify with the module assigned:\n${after.out}`);
// **And removal is the same unit's business.** Unassigned, the host stops it; stopping it takes
// the anchor away and refreshes the bundles, so the machine stops trusting the mesh.
await mesh(`unassign ${MACHINE} ca-trust`);
await mesh(`push ${MACHINE}`);
await settled();
const gone = await on(`test -e ${ANCHOR}`);
assert.equal(gone.ok, false, "the anchor is still on the machine after the module was unassigned");
const afterwards = await verifying();
assert.equal(afterwards.ok, false, `the machine still verifies the mesh's authority with nothing anchoring it:\n${afterwards.out}`);
assert.match(afterwards.out, /certificate|issuer/i, `it failed for some other reason:\n${afterwards.out}`);
});
@@ -36,7 +36,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -49,7 +49,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "catalogue-mqtt"; const SCENARIO = "catalogue-mqtt";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -158,11 +158,101 @@ test("the mesh assigns mosquitto: a run-once step seeds dynsec before the broker
skip, timeout: 1_500_000, skip, timeout: 1_500_000,
}, async () => { }, async () => {
// mosquitto's dynsec config: the plugin refuses to start unless dynamic-security.json holds an // mosquitto's dynsec config: the plugin refuses to start unless dynamic-security.json holds an
// admin client, so the store MUST be seeded first. The catalogue's manifest declares a `run-once` // admin client, so the store MUST be seeded first. The `run-once` bootstrap container is declared
// bootstrap container BEFORE `server` (the broker), reusing the module's runtime image; the host // BEFORE `server` (the broker) and reuses the module's runtime image; the host runs it to
// runs it to completion, then starts the broker. The manifest is the catalogue's own, its runtime // completion, then starts the broker. The runtime `server`/`runtime` shape mirrors the committed
// artifact the image this scenario stocked (novox/hq 04-ISSUES/073). // manifest, with images pinned to what this scenario serves by digest.
const manifest = catalogueModule("mosquitto", held); const mosquittoConf =
"persistence true\n" +
"persistence_location /mosquitto/data\n\n" +
"log_dest stdout\n" +
"log_type warning\n" +
"log_type error\n" +
"log_type notice\n\n" +
"# Every client authenticates; identities and their per-topic ACLs are managed\n" +
"# at runtime by the dynamic security plugin, whose store the plugin itself owns.\n" +
"allow_anonymous false\n" +
"plugin /usr/lib/mosquitto_dynamic_security.so\n" +
"plugin_opt_config_file /mosquitto/data/dynamic-security.json\n\n" +
"# MQTT listener\n" +
"listener 1883\n\n" +
"# MQTT-over-WebSockets listener\n" +
"listener 8081\n" +
"protocol websockets\n";
const manifest = JSON.stringify({
module: "mosquitto",
version: "1",
provides: [{ name: "mqtt-topic", scope: "mesh" }],
serves: { "mqtt-topic": {} },
emits: ["module.mosquitto.topic.provisioned", "module.mosquitto.topic.deprovisioned"],
// The events entrypoint subscribes to its own lifecycle events (an audit log), so it consumes
// them too — declared, or the foundation never makes the queue the runtime binds (ADR 0046).
consumes: ["module.mosquitto.topic.provisioned", "module.mosquitto.topic.deprovisioned"],
receives: { "mqtt-topic": "/var/lib/mosquitto-module/grants/mesh.json" },
grants: { "mqtt-topic": "/var/lib/mosquitto-module/grants" },
"own-secrets": {
admin: "/var/lib/mosquitto-module/admin.secret",
broker: "/var/lib/mesh/mosquitto/broker",
},
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/mosquitto", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/mosquitto-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/mosquitto-module/grants", mode: "0700" },
// The broker runs as uid 1883, so the shared data directory it seeds into and persists to is
// its own.
{ id: "data", type: "directory", path: "/services/mosquitto/data", mode: "0700", owner: "1883:1883" },
{
id: "server-conf", type: "file", path: "/var/lib/mosquitto-module/mosquitto.conf",
mode: "0600", owner: "1883:1883", content: mosquittoConf,
},
{ id: "net", type: "network", name: "mosquitto" },
// THE run-once step: seed dynsec offline, once, before the broker. It reuses the runtime image
// (`mesh-tools run <bootstrap>` imports mosquitto's bootstrap entrypoint, which writes the
// admin client into dynamic-security.json and chowns it to the broker's uid, then exits). It is
// declared BEFORE `server`; the host runs it to completion and requires exit 0 before starting
// the broker.
{
id: "bootstrap", type: "container", name: "mosquitto-bootstrap",
image: pinned("mesh-runtime-mosquitto"), "run-once": true,
volumes: [
"/services/mosquitto/data:/mosquitto/data",
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro",
],
env: {
MESH_PROVISION_MQTT: "mosquitto:1883",
MESH_PROVISION_ADMIN_USER: "mesh-admin",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/admin",
MESH_DYNSEC_FILE: "/mosquitto/data/dynamic-security.json",
},
args: ["run", "/app/modules/mosquitto/dist/bootstrap/index.js"],
},
{
id: "server", type: "container", name: "mosquitto", image: pinned("eclipse-mosquitto"),
network: "mosquitto", ports: ["1883", "8081"],
volumes: [
"/services/mosquitto/data:/mosquitto/data",
"/var/lib/mosquitto-module/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro",
],
},
{
id: "runtime", type: "container", name: "mesh-mosquitto",
image: pinned("mesh-runtime-mosquitto"), network: "mosquitto",
volumes: [
"/var/lib/mesh/mosquitto/broker:/run/secrets/broker:ro",
"/var/lib/mosquitto-module/grants:/var/lib/mosquitto-module/grants:ro",
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/mosquitto-module/grants/mesh.json",
MESH_PROVISION_MQTT: "mosquitto:1883",
MESH_PROVISION_ADMIN_USER: "mesh-admin",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/admin",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/mosquitto.json && docker cp /tmp/mosquitto.json mesh-controller:/mosquitto.json`); await must(`printf %s ${quote(manifest)} > /tmp/mosquitto.json && docker cp /tmp/mosquitto.json mesh-controller:/mosquitto.json`);
await mesh("module add /mosquitto.json"); await mesh("module add /mosquitto.json");
+210
View File
@@ -0,0 +1,210 @@
/**
* The mesh assigns grafana's tool runtime, configured entirely by the assignment's settings — the
* ADR 0051 + 0052 case: config is the assignment's, delivered as a settings-merged file the runtime
* reads, not a credential baked into the manifest.
*
* plex/sonarr prove a runtime that self-detects its key from the app's own config. This proves the
* other half: the operator states grafana's URL and an API token as settings for this node, the
* control plane merges them into the module's mergeable config file, and the runtime reads that file
* at start, registers grafana's tools, and serves them under its scoped account. There is no live
* Grafana — that the serve queue is bound is the proof the settings reached the runtime and its
* tools loaded from them.
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh grafana builds mesh-runtime-grafana:development into the local
* daemon, which scenarios/grafana-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "grafana-node";
const MACHINE = "anchor";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh assigns grafana's runtime, configured by settings, and it serves its tools", {
skip, timeout: 900_000,
}, async () => {
// A grafana manifest with no credential in it: its runtime, and a mergeable config file the
// settings will fill. This is the whole point of ADR 0051 — the manifest carries defaults and
// structure, the assignment carries the URL and token.
const manifest = JSON.stringify({
module: "grafana",
version: "1",
emits: ["module.grafana.alert.firing"],
"own-secrets": { broker: "/var/lib/mesh/grafana/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/grafana", mode: "0700" },
{ id: "config", type: "file", path: "/var/lib/mesh/grafana/config.json", mode: "0600", content: "{}\n", merge: "json" },
{
id: "runtime", type: "container", name: "mesh-grafana", image: pinned("mesh-runtime-grafana"),
network: "host",
volumes: [
"/var/lib/mesh/grafana/broker:/run/secrets/broker:ro",
"/var/lib/mesh/grafana/config.json:/run/config/config.json:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_GRAFANA_CONFIG_FILE: "/run/config/config.json",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/grafana.json && docker cp /tmp/grafana.json mesh-controller:/grafana.json`);
await mesh("module add /grafana.json");
// The operator states grafana's URL and API token as settings for this node — the config the
// runtime will read. Nothing about them is in the manifest.
const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token: "lab-grafana-token" });
await must(`printf %s ${quote(settings)} > /tmp/grafana-settings.json && docker cp /tmp/grafana-settings.json mesh-controller:/grafana-settings.json`);
await mesh(`settings set grafana /grafana-settings.json --node ${MACHINE}`);
const issued = await mesh(`module issue grafana --node ${MACHINE}`);
assert.match(issued, /scoped to what it emits and consumes/, issued);
await mesh(`assign ${MACHINE} grafana`);
await mesh(`push ${MACHINE}`);
await settled();
const running = await must(`docker ps --format '{{.Names}}'`);
assert.match(running, /mesh-grafana/,
`grafana's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
// The settings reached the node: the rendered config file carries what was set, not the manifest's
// empty default.
const config = await must(`cat /var/lib/mesh/grafana/config.json`);
assert.match(config, /lab-grafana-token/, `the settings did not merge into the config file:\n${config}`);
const credential = await must(`cat /var/lib/mesh/grafana/broker`);
assert.match(credential, /"url":"amqps:\/\/anchor-grafana:/, `not the scoped account:\n${credential}`);
assert.doesNotMatch(credential, /guest:guest/, "grafana's runtime holds the broker's own account");
// The runtime read that config, built its client from the settings-provided token, registered its
// tools, and bound their serve queues — the queue on the broker is the proof the settings-config
// path reached serving, with no credential in the manifest and no live Grafana.
let served = "";
const untilServing = Date.now() + 60_000;
while (Date.now() < untilServing) {
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
if (/serve\.grafana\.grafana_status/.test(served)) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(served, /serve\.grafana\.grafana_status/,
`grafana's runtime never bound its serve queue (settings not read?):\n` +
`${(await on(`docker logs mesh-grafana 2>&1 | tail -20`)).out}\n---\n${served}`);
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
assert.match(users, /anchor-grafana/, `the scoped account is not on the broker:\n${users}`);
});
+42 -11
View File
@@ -38,7 +38,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -51,7 +51,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "model-usage-bed"; const SCENARIO = "model-usage-bed";
/** The node that carries the postgres provider and the model-usage consumer. anchor carries only the /** The node that carries the postgres provider and the model-usage consumer. anchor carries only the
@@ -190,9 +190,7 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot
}, async () => { }, async () => {
// ================================================================================================ // ================================================================================================
// THE MANIFESTS — postgres verbatim from two-node-db (its server publishes 5432 so the consumer // THE MANIFESTS — postgres verbatim from two-node-db (its server publishes 5432 so the consumer
// reaches it): a SECOND postgres beside the foundation's store, which the catalogue's postgres would // reaches it), and model-usage the committed catalogue shape with its images pinned.
// instead claim and adopt in place. Still an inline copy, declared in beds-read-the-catalogue.test.ts
// (novox/hq 04-ISSUES/073). model-usage is the catalogue's.
// ================================================================================================ // ================================================================================================
const postgresManifest = JSON.stringify({ const postgresManifest = JSON.stringify({
module: "postgres", module: "postgres",
@@ -235,12 +233,45 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot
], ],
}); });
// --- model-usage: the catalogue's own manifest (novox/hq 04-ISSUES/073). It requires // --- model-usage: requires postgres-database, owns a provisioned store, consumes module.*.usage.*,
// postgres-database, owns a provisioned store, consumes module.*.usage.*, and its runtime is on the // runs a run-once migrate then the long-lived event consumer. Both containers on the host network so
// host network so it reaches the granted postgres (at the provider's address the mesh writes) and // they reach the granted postgres (at the provider's address the mesh writes) and the broker. ------
// the broker. Its slug keeps the consumer identity under the 20 characters an S3 access key allows const modelUsageManifest = JSON.stringify({
// (ADR 0049). ------------------------------------------------------------------------------------ module: "model-usage",
const modelUsageManifest = catalogueModule("model-usage", held); version: "1",
// `mesh_laptop_model-usage` is 23 chars, over the 20 an S3 access key keeps (ADR 0049); a short
// slug makes the consumer identity `mesh_laptop_usage` (17). db/role/`as` all derive from it.
slug: "usage",
capabilities: ["container-runtime"],
requires: ["postgres-database"],
contributes: { "postgres-database": { name: "model_usage" } },
binds: { "postgres-database": "/var/lib/model-usage/database.json" },
secrets: { "postgres-database": "/var/lib/model-usage/database.secret" },
consumes: ["module.*.usage.*"],
"own-secrets": { broker: "/var/lib/mesh/model-usage/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/model-usage", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/model-usage", mode: "0700" },
// The connection string carries the password, so it reaches the runtime as a file the mesh
// templates (novox/hq ADR 0086), the shape the catalogue's manifest has.
{
id: "database-url", type: "file", path: "/var/lib/model-usage/database.url", mode: "0600",
content:
"postgresql://${bound:postgres-database:as}:${secret:postgres-database}@" +
"${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n",
},
{
id: "runtime", type: "container", name: "mesh-model-usage",
image: pinned("mesh-runtime-model-usage"), network: "host",
volumes: [
"/var/lib/mesh/model-usage/broker:/run/secrets/broker:ro",
"/var/lib/model-usage:/run/state",
"/var/lib/model-usage/database.url:/run/secrets/database-url:ro",
],
env: { MESH_BROKER_FILE: "/run/secrets/broker", DATABASE_URL_FILE: "/run/secrets/database-url" },
},
],
});
async function addIssueAssign(name: string, manifest: string): Promise<void> { async function addIssueAssign(name: string, manifest: string): Promise<void> {
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
+231
View File
@@ -0,0 +1,231 @@
/**
* The mesh assigns plex's tool runtime, and it serves plex's tools over an account the mesh
* delivered — the whole of novox/hq ADR 0052.
*
* assigned-audit proves an assigned *consumer* (ADR 0048). This proves an assigned module that runs
* its OWN code as its OWN process under its OWN scoped account and *serves tools*: the module is
* assigned through the control plane, the mesh issues it an account scoped to serve.plex.* (and its
* events), seals it to the machine, and the host runs it as a container that binds amqps with that
* account. A caller then invokes plex.plex_reachable over the mesh and gets the tool's own answer —
* proof the invocation routed to the assigned runtime, ran plex's real code, and replied, all under
* the scoped account and never the broker's own.
*
* It needs the host binary, the foundation bundle, and the runtime image the scenario loads:
*
* MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh plex builds mesh-runtime-plex:development into the local daemon,
* which scenarios/plex-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "plex-node";
const MACHINE = "anchor";
let instanceId = "";
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
/** The control plane, a container on the node. */
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
// Raise the foundation — store, broker, control — from the bundle.
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
// The node joins its own mesh, so it is a node the mesh can assign to, and start the host so it
// applies what it is pushed.
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh assigns plex's runtime, and it serves plex's tools over the account the mesh delivered", {
skip, timeout: 900_000,
}, async () => {
// A minimal plex manifest: its tools/events runtime (no Plex server or media mounts in the lab),
// its emits and consumes so the account is scoped to those too, and a token in the environment so
// the tools register without a running Plex to detect one from. The runtime image is the digest
// this scenario's registry serves.
const manifest = JSON.stringify({
module: "plex",
version: "1",
emits: [
"module.plex.playback.started",
"module.plex.playback.stopped",
"module.plex.item.added",
],
consumes: ["module.*.download.completed"],
"own-secrets": { broker: "/var/lib/mesh/plex/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/plex", mode: "0700" },
{
id: "runtime", type: "container", name: "mesh-plex", image: pinned("mesh-runtime-plex"),
network: "host",
volumes: ["/var/lib/mesh/plex/broker:/run/secrets/broker:ro"],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_PLEX_URL: "http://127.0.0.1:32400",
MESH_PLEX_TOKEN: "lab-token",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/plex.json && docker cp /tmp/plex.json mesh-controller:/plex.json`);
await mesh("module add /plex.json");
// The mesh issues plex's scoped account and seals it to this machine, then assigns and pushes it.
const issued = await mesh(`module issue plex --node ${MACHINE}`);
assert.match(issued, /scoped to what it emits and consumes/, issued);
await mesh(`assign ${MACHINE} plex`);
await mesh(`push ${MACHINE}`);
await settled();
// The runtime container the mesh started is running.
const running = await must(`docker ps --format '{{.Names}}'`);
assert.match(running, /mesh-plex/,
`plex's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
// The credential on disk is the scoped account over amqps, sealed — not the broker's own.
const credential = await must(`cat /var/lib/mesh/plex/broker`);
assert.match(credential, /"url":"amqps:\/\/anchor-plex:/, `not the scoped account:\n${credential}`);
assert.doesNotMatch(credential, /guest:guest/, "plex's runtime holds the broker's own account");
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker");
// The runtime registered and is serving its tools — the queue it declared is on the broker,
// named for the scope its account is granted (serve.plex.*).
let served = "";
const untilServing = Date.now() + 60_000;
while (Date.now() < untilServing) {
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
if (/serve\.plex\.plex_reachable/.test(served)) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(served, /serve\.plex\.plex_reachable/,
`plex's runtime never bound its serve queue:\n${(await on(`docker logs mesh-plex 2>&1 | tail -20`)).out}\n---\n${served}`);
// A caller invokes plex.plex_reachable over the mesh, from the bootstrap account (a caller, like
// mesh-controller's command API — plex's own account serves, it does not call). The reply is the
// tool's own answer: it ran in the assigned runtime and reported the Plex server is unreachable
// (there is none in the lab). A reply at all — not a timeout — is the proof the invocation routed
// to the assigned runtime and ran plex's real code under its scoped account.
const invoked = await must(
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
`${pinned("mesh-runtime-plex")} invoke plex plex_reachable`,
120_000,
);
const line = invoked.split("\n").map((l) => l.trim()).filter(Boolean).pop() ?? "";
const result = JSON.parse(line) as { reachable: boolean; url: string; error?: string };
assert.equal(result.reachable, false, `expected the lab's Plex to be unreachable:\n${invoked}`);
assert.match(result.url, /127\.0\.0\.1:32400/, `the tool ran but not against the configured server:\n${invoked}`);
// And the account the mesh made for it is a real one on the broker, scoped — proven above by the
// serve queue authenticating and the invocation round-tripping under it.
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
assert.match(users, /anchor-plex/, `the scoped account is not on the broker:\n${users}`);
});
+273
View File
@@ -0,0 +1,273 @@
/**
* The mesh assigns redis — a *provider* — and its runtime runs the provisioner AND the tools as one
* process under one account the mesh delivered (novox/hq ADR 0052).
*
* assigned-plex proves an assigned module that serves tools. This proves the provider half: redis's
* runtime binds its scoped account, serves redis's tools against the real server (redis_ping →
* PONG), and — the thing 0052 fixes — runs its provisioner in that same broker-bound process, so a
* grant is provisioned and its lifecycle event is emitted onto the mesh. Before 0052 the provisioner
* ran in a container with no broker and its emit could not fire at all.
*
* A caveat this test makes explicit: runProvisioner needs a seal key ($MESH_SEAL_KEY) and the mesh
* has no way yet to deliver one to a provider's runtime (04-ISSUES). The manifest here sets a
* lab-local key so the mechanism can be proven; the delivery is a separate, open design question.
*
* It needs the host binary, the foundation bundle, and the runtime image the scenario loads:
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development into the local
* daemon, which scenarios/redis-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "redis-node";
const MACHINE = "anchor";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh assigns redis, and its runtime serves tools and provisions grants over the account the mesh delivered", {
skip, timeout: 900_000,
}, async () => {
// A redis manifest with both halves it needs on this node: the redis server, and one broker-bound
// runtime that serves redis's tools AND runs its provisioner. Both reach the server over the host
// (127.0.0.1:6379) with the same admin password the mesh generated. MESH_SEAL_KEY is lab-local —
// the mesh cannot yet deliver one to a provider's runtime (see the file header / 04-ISSUES).
const manifest = JSON.stringify({
module: "redis",
version: "1",
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
// redis's events entrypoint subscribes to its own lifecycle events (an audit-trail log), so it
// consumes them too — declared, or the foundation never makes the queue the runtime binds and it
// crashes on start with a 404 (novox/hq ADR 0046: a consume is declared).
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
{
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
},
{
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
volumes: [
"/services/redis/data:/data",
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
],
args: ["/etc/redis/redis.conf"],
},
{
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
network: "host",
volumes: [
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
GRANTS: "/var/lib/redis-module/grants",
MESH_PROVISION_REDIS: "127.0.0.1:6379",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
MESH_SEAL_KEY: "lab-only-seal-key",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
await mesh("module add /redis.json");
const issued = await mesh(`module issue redis --node ${MACHINE}`);
assert.match(issued, /scoped to what it emits and consumes/, issued);
await mesh(`assign ${MACHINE} redis`);
await mesh(`push ${MACHINE}`);
await settled();
// The server and the runtime the mesh started are both running.
const running = await must(`docker ps --format '{{.Names}}'`);
assert.match(running, /\bredis\b/, `redis's server is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
assert.match(running, /mesh-redis/, `redis's runtime is not running:\n${(await on(`docker logs mesh-redis 2>&1 | tail -20`)).out}`);
// The credential on disk is the scoped account over amqps, sealed — not the broker's own.
const credential = await must(`cat /var/lib/mesh/redis/broker`);
assert.match(credential, /"url":"amqps:\/\/anchor-redis:/, `not the scoped account:\n${credential}`);
assert.doesNotMatch(credential, /guest:guest/, "redis's runtime holds the broker's own account");
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker");
// The runtime registered and is serving its tools — the serve queue is on the broker.
let served = "";
const untilServing = Date.now() + 60_000;
while (Date.now() < untilServing) {
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
if (/serve\.redis\.redis_ping/.test(served)) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(served, /serve\.redis\.redis_ping/,
`redis's runtime never bound its serve queue:\n${(await on(`docker logs mesh-redis 2>&1 | tail -30`)).out}\n---\n${served}`);
// A caller invokes redis.redis_ping over the mesh: the tool runs in the assigned runtime, reaches
// the real redis, and answers PONG. A positive round-trip against a real backend.
const pinged = await must(
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
`${pinned("mesh-runtime-redis")} invoke redis redis_ping`,
120_000,
);
const pingLine = pinged.split("\n").map((l) => l.trim()).filter(Boolean).pop() ?? "";
const ping = JSON.parse(pingLine) as { ok: unknown };
assert.ok(String(ping.ok).toUpperCase().includes("PONG") || ping.ok === true,
`redis_ping did not answer PONG through the mesh:\n${pinged}`);
// The provider path: a grant appears (as the control plane would write it), and the provisioner —
// running inside the same broker-bound runtime — creates the ACL user and emits the lifecycle
// event. The sealed credential the harness writes only after adapter.create() returns is the
// proof create() ran to completion; and because emit() awaits the broker's publish confirm
// (ADR 0047), a completed create() means the provisioned event was accepted onto the mesh.
const grant = JSON.stringify({ resource: "redis-cache", consumer: "app-one", node: MACHINE, values: {} });
await must(`printf %s ${quote(grant)} > /var/lib/redis-module/grants/app-one.grant.json`);
let credentialWritten = false;
const untilProvisioned = Date.now() + 60_000;
while (Date.now() < untilProvisioned) {
const ls = await on(`ls /var/lib/redis-module/grants/`);
if (ls.ok && /app-one\.redis-cache\.credential/.test(ls.out)) { credentialWritten = true; break; }
await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(credentialWritten,
`the provisioner never provisioned the grant (no emit under a bound broker?):\n` +
`${(await on(`docker logs mesh-redis 2>&1 | tail -30`)).out}`);
// No emit failed: the provisioner's announce() logs "emit ... failed" only when the broker refused
// the publish. Its absence, with the credential written, is the provisioner emitting on the mesh.
const runtimeLog = (await on(`docker logs mesh-redis 2>&1`)).out;
assert.doesNotMatch(runtimeLog, /emit .*failed/,
`the provisioner's emit was refused — the account cannot publish its lifecycle event:\n${runtimeLog}`);
// And the ACL user the provisioner created is really on the redis server — the provisioning did
// its own half, not only the mesh bookkeeping. Asked through the same served tool surface.
const acl = await must(
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
`${pinned("mesh-runtime-redis")} invoke redis redis_command '{"command":"ACL LIST"}'`,
120_000,
);
assert.match(acl, /app-one/, `the provisioner did not create the consumer's ACL user on redis:\n${acl}`);
// The scoped account the mesh made for it is a real one on the broker.
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
assert.match(users, /anchor-redis/, `the scoped account is not on the broker:\n${users}`);
});
+214
View File
@@ -0,0 +1,214 @@
/**
* The mesh assigns sonarr's tool runtime, and it serves sonarr's tools over an account the mesh
* delivered — the Servarr case of novox/hq ADR 0052.
*
* assigned-plex proved a tools+events module that self-detects its token from a mounted config dir.
* This proves that self-configuring pattern generalises to the Servarr family: sonarr's runtime
* detects its API key from the server's own config.xml (a file resource stands in for the running
* Sonarr here), registers its tools, and serves them under a scoped account. There is no live Sonarr
* to reach — that the serve queue is bound is the proof the key was detected and the tools loaded.
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh sonarr builds mesh-runtime-sonarr:development into the local
* daemon, which scenarios/sonarr-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "sonarr-node";
const MACHINE = "anchor";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh assigns sonarr's runtime, and it detects its key and serves its tools", {
skip, timeout: 900_000,
}, async () => {
// A minimal sonarr manifest: its tool runtime, and a config.xml the runtime detects its API key
// from — the file resource stands in for the running Sonarr that would write it. No Sonarr server
// or media mounts; the tools simply have nothing live to reach.
const manifest = JSON.stringify({
module: "sonarr",
version: "1",
emits: ["module.sonarr.episode.grabbed", "module.sonarr.download.completed"],
consumes: [],
"own-secrets": { broker: "/var/lib/mesh/sonarr/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/sonarr", mode: "0700" },
{ id: "config", type: "directory", path: "/services/sonarr/config", mode: "0700" },
{
id: "config-xml", type: "file", path: "/services/sonarr/config/config.xml", mode: "0644",
content: "<Config>\n <Port>8989</Port>\n <ApiKey>labdetectedapikey0000000000000000</ApiKey>\n</Config>\n",
},
{
id: "runtime", type: "container", name: "mesh-sonarr", image: pinned("mesh-runtime-sonarr"),
network: "host",
volumes: [
"/var/lib/mesh/sonarr/broker:/run/secrets/broker:ro",
"/services/sonarr/config:/var/lib/sonarr/config:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_SONARR_URL: "http://127.0.0.1:8989",
MESH_SONARR_CONFIG_DIR: "/var/lib/sonarr/config",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/sonarr.json && docker cp /tmp/sonarr.json mesh-controller:/sonarr.json`);
await mesh("module add /sonarr.json");
const issued = await mesh(`module issue sonarr --node ${MACHINE}`);
assert.match(issued, /scoped to what it emits and consumes/, issued);
await mesh(`assign ${MACHINE} sonarr`);
await mesh(`push ${MACHINE}`);
await settled();
const running = await must(`docker ps --format '{{.Names}}'`);
assert.match(running, /mesh-sonarr/,
`sonarr's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
const credential = await must(`cat /var/lib/mesh/sonarr/broker`);
assert.match(credential, /"url":"amqps:\/\/anchor-sonarr:/, `not the scoped account:\n${credential}`);
assert.doesNotMatch(credential, /guest:guest/, "sonarr's runtime holds the broker's own account");
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker");
// The runtime detected its API key from config.xml, registered its tools, and bound their serve
// queues — the queue on the broker is the proof the whole chain worked with no live Sonarr.
let served = "";
const untilServing = Date.now() + 60_000;
while (Date.now() < untilServing) {
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
if (/serve\.sonarr\.sonarr_status/.test(served)) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(served, /serve\.sonarr\.sonarr_status/,
`sonarr's runtime never bound its serve queue (key not detected?):\n` +
`${(await on(`docker logs mesh-sonarr 2>&1 | tail -20`)).out}\n---\n${served}`);
// A caller invokes sonarr_status over the mesh: it routes to the assigned runtime, which runs
// sonarr's real code and reports Sonarr unreachable (there is none). A reply — not a timeout — is
// the proof the invocation reached the runtime under its scoped account.
const invoked = await on(
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
`${pinned("mesh-runtime-sonarr")} invoke sonarr sonarr_status`,
120_000,
);
assert.doesNotMatch(invoked.out, /timed out/,
`sonarr_status timed out — nothing served the invocation:\n${invoked.out}`);
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
assert.match(users, /anchor-sonarr/, `the scoped account is not on the broker:\n${users}`);
});
@@ -29,7 +29,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -42,7 +42,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "tools-confluence"; const SCENARIO = "tools-confluence";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -155,9 +155,35 @@ test("the mesh assigns confluence: its tools-only runtime comes up and serves th
// confluence is tools-only and outbound-only: no service, no listener, no provisioner — just a // confluence is tools-only and outbound-only: no service, no listener, no provisioner — just a
// broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the // broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the
// lab has no real Confluence, so the token points at nothing — and that is the case under test: the // lab has no real Confluence, so the token points at nothing — and that is the case under test: the
// runtime must serve every tool regardless. The manifest is the catalogue's own, its runtime // runtime must serve every tool regardless. The runtime container name and shape mirror the
// artifact the image this scenario stocked (novox/hq 04-ISSUES/073). // committed manifest, with the image pinned to what this scenario serves by digest.
const manifest = catalogueModule("confluence", held); const manifest = JSON.stringify({
module: "confluence",
version: "1",
"own-secrets": {
token: "/var/lib/confluence/token",
broker: "/var/lib/mesh/confluence/broker",
},
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/confluence", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/confluence", mode: "0700" },
{ id: "config", type: "file", path: "/var/lib/confluence/config.json", merge: "json", content: "{}", mode: "0600" },
{
id: "runtime", type: "container", name: "mesh-runtime-confluence",
image: pinned("mesh-runtime-confluence"), network: "host",
volumes: [
"/var/lib/confluence/config.json:/run/config/config.json:ro",
"/var/lib/confluence/token:/run/secrets/token:ro",
"/var/lib/mesh/confluence/broker:/run/secrets/broker:ro",
],
env: {
MESH_CONFLUENCE_TOKEN_FILE: "/run/secrets/token",
MESH_CONFLUENCE_CONFIG_FILE: "/run/config/config.json",
MESH_BROKER_FILE: "/run/secrets/broker",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/confluence.json && docker cp /tmp/confluence.json mesh-controller:/confluence.json`); await must(`printf %s ${quote(manifest)} > /tmp/confluence.json && docker cp /tmp/confluence.json mesh-controller:/confluence.json`);
await mesh("module add /confluence.json"); await mesh("module add /confluence.json");
@@ -204,15 +230,6 @@ test("the mesh assigns confluence: its tools-only runtime comes up and serves th
assert.match(served2, /serve\.confluence\.confluence_search/, assert.match(served2, /serve\.confluence\.confluence_search/,
`confluence's runtime never bound its serve queue:\n${(await on(`docker logs mesh-runtime-confluence 2>&1 | tail -20`)).out}\n---\n${served2}`); `confluence's runtime never bound its serve queue:\n${(await on(`docker logs mesh-runtime-confluence 2>&1 | tail -20`)).out}\n---\n${served2}`);
// --- and the control plane is the way to ask it (novox/hq ADR 0095, issue 049) ------------------
// No account in the mesh but the control plane's may create a reply queue and publish to a
// module's request key. Asked through it, the tool ANSWERS — with an error, since the lab has no
// Confluence and no token, which is an answer: the round trip is what is under test, and a
// timeout would read differently.
const asked = await on(`docker exec mesh-controller /mesh-controller ask confluence confluence_search '{"query":"mesh"}' --wait 60s`, 90_000);
assert.doesNotMatch(asked.out, /did not answer/, `the tool was never reached through the control plane:\n${asked.out}`);
assert.match(asked.out, /"(result|error)"/, `the control plane printed no answer:\n${asked.out}`);
// --- confluence got its own scoped broker account ----------------------------------------------- // --- confluence got its own scoped broker account -----------------------------------------------
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`); const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
assert.match(users, /anchor-confluence/, `the scoped account anchor-confluence is not on the broker:\n${users}`); assert.match(users, /anchor-confluence/, `the scoped account anchor-confluence is not on the broker:\n${users}`);
+31 -5
View File
@@ -28,7 +28,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -41,7 +41,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "tools-gitlab"; const SCENARIO = "tools-gitlab";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -154,9 +154,35 @@ test("the mesh assigns gitlab: its tools-only runtime comes up and serves the fu
// gitlab is tools-only and outbound-only: no service, no listener, no provisioner — just a // gitlab is tools-only and outbound-only: no service, no listener, no provisioner — just a
// broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the // broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the
// lab has no real GitLab, so the token points at nothing — and that is the case under test: the // lab has no real GitLab, so the token points at nothing — and that is the case under test: the
// runtime must serve every tool regardless. The manifest is the catalogue's own, its runtime // runtime must serve every tool regardless. The runtime container name and shape mirror the
// artifact the image this scenario stocked (novox/hq 04-ISSUES/073). // committed manifest, with the image pinned to what this scenario serves by digest.
const manifest = catalogueModule("gitlab", held); const manifest = JSON.stringify({
module: "gitlab",
version: "1",
"own-secrets": {
token: "/var/lib/gitlab/token",
broker: "/var/lib/mesh/gitlab/broker",
},
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/gitlab", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/gitlab", mode: "0700" },
{ id: "config", type: "file", path: "/var/lib/gitlab/config.json", merge: "json", content: "{}", mode: "0600" },
{
id: "runtime", type: "container", name: "mesh-runtime-gitlab",
image: pinned("mesh-runtime-gitlab"), network: "host",
volumes: [
"/var/lib/gitlab/config.json:/run/config/config.json:ro",
"/var/lib/gitlab/token:/run/secrets/token:ro",
"/var/lib/mesh/gitlab/broker:/run/secrets/broker:ro",
],
env: {
MESH_GITLAB_TOKEN_FILE: "/run/secrets/token",
MESH_GITLAB_CONFIG_FILE: "/run/config/config.json",
MESH_BROKER_FILE: "/run/secrets/broker",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/gitlab.json && docker cp /tmp/gitlab.json mesh-controller:/gitlab.json`); await must(`printf %s ${quote(manifest)} > /tmp/gitlab.json && docker cp /tmp/gitlab.json mesh-controller:/gitlab.json`);
await mesh("module add /gitlab.json"); await mesh("module add /gitlab.json");
+107 -61
View File
@@ -8,14 +8,15 @@
* *
* This bed proves that across two machines. `anchor` is the control-node: it raises the foundation * This bed proves that across two machines. `anchor` is the control-node: it raises the foundation
* and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner. * and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner.
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database`. Each * `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database` — plus
* consumer's database is minted on the store on anchor and reached over the overlay: their bindings * a co-located `redis` (which holds no seat). Each consumer's database is minted on the store on
* name `anchor.internal`, and their minted logins authenticate against the store. baserow's cache is * anchor and reached over the overlay: their bindings name `anchor.internal`, and their minted logins
* its own, inside its container (novox/hq 081). * authenticate against the store. redis stays co-located on laptop for baserow's cache.
* *
* The three manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim * The four manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, and baserow/letta * from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, redis runs on
* wire their servers to the grant the mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed * the host network with a lab-local seal key, and baserow/letta wire their servers to the grant the
* mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
* ONCE; laptop converges once with every one up, and the two consumers are provisioned against the * ONCE; laptop converges once with every one up, and the two consumers are provisioned against the
* database the provider on their own node gave them. * database the provider on their own node gave them.
* *
@@ -24,23 +25,25 @@
* MESH_LAB_HOST_BINARY=.../mesh-host * MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock * MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* *
* HELPER — stock the three runtimes into the local daemon before the run (some may already be there): * HELPER — stock the four runtimes into the local daemon before the run (some may already be there):
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar * scripts/build-module-runtime.sh postgres /tmp/postgres.tar
* scripts/build-module-runtime.sh redis /tmp/redis.tar
* scripts/build-module-runtime.sh baserow /tmp/baserow.tar * scripts/build-module-runtime.sh baserow /tmp/baserow.tar
* scripts/build-module-runtime.sh letta /tmp/letta.tar * scripts/build-module-runtime.sh letta /tmp/letta.tar
* The service images (postgres, baserow, letta, each pinned by digest) * The service images (postgres:17-alpine, redis:7-alpine, baserow/baserow:latest, letta/letta:latest)
* must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls * must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls
* what it runs from the scenario's own registry by digest. * what it runs from the scenario's own registry by digest.
*/ */
import { test, before, after } from "node:test"; import { test, before, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync } from "node:fs"; import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts"; import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -53,13 +56,16 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "two-node-db"; const SCENARIO = "two-node-db";
/** The node that carries the whole DB-consumer chain. anchor carries only the foundation. */ /** The node that carries the whole DB-consumer chain. anchor carries only the foundation. */
const NODE = "laptop"; const NODE = "laptop";
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
const catalogDir = process.env["MESH_LAB_CATALOG"]
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
let instanceId = ""; let instanceId = "";
/** The mesh's own images, as the machines hold them. */ /** The mesh's own images, as the machines hold them. */
@@ -150,13 +156,12 @@ async function settled(node: string, withinMs = 1_200_000): Promise<void> {
last = said; last = said;
await new Promise((r) => setTimeout(r, 5000)); await new Promise((r) => setTimeout(r, 5000));
} }
// Timed out — capture what the node that did not answer is doing, so the failure is diagnosable. // Timed out — capture what the node is actually doing so the failure is diagnosable.
// Its own machine, not the second node's: the anchor timing out used to print the laptop's log. const ps = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; const hostLog = (await on(NODE, `tail -80 /var/log/mesh-host.log`)).out;
const hostLog = (await on(node, `tail -80 /var/log/mesh-host.log`)).out;
throw new Error( throw new Error(
`${node} never caught up within ${Math.round(withinMs / 1000)}s.\nLast status:\n${last}\n` + `${node} never caught up within ${Math.round(withinMs / 1000)}s.\nLast status:\n${last}\n` +
`--- ${node} docker ps -a ---\n${ps}\n--- ${node} mesh-host.log tail ---\n${hostLog}`); `--- ${NODE} docker ps -a ---\n${ps}\n--- ${NODE} mesh-host.log tail ---\n${hostLog}`);
} }
before(async () => { before(async () => {
@@ -204,38 +209,82 @@ test("consumers on a joined node get their databases from the one foundation sto
// they land on changes. // they land on changes.
// ================================================================================================ // ================================================================================================
// --- baserow: a consumer that requires postgres-database, its server wired to the grant the mesh // --- redis: a cache provider on the host network (127.0.0.1:6379). No seal key: the provider
// writes, plus a runtime that serves baserow's tools. Its cache is its own — the image runs one when // is handed the minted credential already unsealed by the host (ADR 0048). It carries
// no REDIS_HOST is given — because baserow keeps keys and channels under fixed names a shared // the committed provides/serves/receives/grants so baserow's redis-cache requirement resolves. ----
// cache's per-consumer grant cannot confine (novox/hq 081). -------------------------------------- const redisManifest = JSON.stringify({
module: "redis",
version: "1",
provides: [{ name: "redis-cache", scope: "mesh" }],
serves: { "redis-cache": { port: 6379 } },
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
grants: { "redis-cache": "/var/lib/redis-module/grants" },
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
{
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
},
{
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
volumes: [
"/services/redis/data:/data",
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
],
args: ["/etc/redis/redis.conf"],
},
{
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
network: "host",
volumes: [
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
GRANTS: "/var/lib/redis-module/grants",
MESH_PROVISION_REDIS: "127.0.0.1:6379",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
},
},
],
});
// --- baserow: a consumer that requires postgres-database AND redis-cache; server wired to both
// from the grants the mesh writes, plus a runtime that serves baserow's tools. --------------------
const baserowManifest = JSON.stringify({ const baserowManifest = JSON.stringify({
module: "baserow", module: "baserow",
version: "1", version: "1",
requires: ["postgres-database"], requires: ["postgres-database", "redis-cache"],
contributes: { "postgres-database": { name: "baserow" } }, contributes: { "postgres-database": { name: "baserow" } },
binds: { "postgres-database": "/var/lib/baserow/database.json" }, binds: { "postgres-database": "/var/lib/baserow/database.json", "redis-cache": "/var/lib/baserow/redis.json" },
secrets: { "postgres-database": "/var/lib/baserow/database.secret" }, secrets: { "postgres-database": "/var/lib/baserow/database.secret", "redis-cache": "/var/lib/baserow/redis.secret" },
"own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" }, "own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" },
resources: [ resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" }, { id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/baserow", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/baserow", mode: "0700" },
// 0755, as the image ships it: the cache it runs for itself does so as another user, who { id: "data", type: "directory", path: "/services/baserow/data", mode: "0700", owner: "9999:9999" },
// must be able to reach its own directory under this one (novox/hq 081).
{ id: "data", type: "directory", path: "/services/baserow/data", mode: "0755", owner: "9999:9999" },
{ {
id: "server-env", type: "file", path: "/var/lib/baserow/server.env", mode: "0600", id: "server-env", type: "file", path: "/var/lib/baserow/server.env", mode: "0600",
content: content:
"DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" + "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" +
"DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" + "DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" +
"DATABASE_PASSWORD=${secret:postgres-database}\n" + "DATABASE_PASSWORD=${secret:postgres-database}\nREDIS_HOST=${bound:redis-cache:at}\n" +
"REDIS_PORT=${bound:redis-cache:port}\nREDIS_PROTOCOL=redis\nREDIS_PASSWORD=${secret:redis-cache}\n" +
"SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n", "SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n",
}, },
{ id: "net", type: "network", name: "baserow" }, { id: "net", type: "network", name: "baserow" },
{ {
id: "server", type: "container", name: "baserow", image: pinned("baserow/baserow"), network: "baserow", id: "server", type: "container", name: "baserow", image: pinned("baserow/baserow"), network: "baserow",
"env-file": ["/var/lib/baserow/server.env"], "env-file": ["/var/lib/baserow/server.env"],
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
"secrets-in-environment": "baserow reads DATABASE_PASSWORD and SECRET_KEY with os.getenv and has no _FILE twin (settings/base.py); not convertible",
volumes: ["/services/baserow/data:/baserow/data"], volumes: ["/services/baserow/data:/baserow/data"],
}, },
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/baserow/config.json", mode: "0600", content: "{}\n", merge: "json" }, { id: "runtime-config", type: "file", path: "/var/lib/mesh/baserow/config.json", mode: "0600", content: "{}\n", merge: "json" },
@@ -280,8 +329,6 @@ test("consumers on a joined node get their databases from the one foundation sto
{ {
id: "server", type: "container", name: "letta", image: pinned("letta/letta"), network: "letta", id: "server", type: "container", name: "letta", image: pinned("letta/letta"), network: "letta",
"env-file": ["/var/lib/letta/server.env"], "env-file": ["/var/lib/letta/server.env"],
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
}, },
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/letta/config.json", mode: "0600", content: "{}\n", merge: "json" }, { id: "runtime-config", type: "file", path: "/var/lib/mesh/letta/config.json", mode: "0600", content: "{}\n", merge: "json" },
{ id: "runtime-env", type: "file", path: "/var/lib/letta/runtime.env", mode: "0600", content: "MESH_LETTA_PASSWORD=${secret:server-password}\n" }, { id: "runtime-env", type: "file", path: "/var/lib/letta/runtime.env", mode: "0600", content: "MESH_LETTA_PASSWORD=${secret:server-password}\n" },
@@ -298,8 +345,6 @@ test("consumers on a joined node get their databases from the one foundation sto
MESH_LETTA_CONFIG_FILE: "/run/config/config.json", MESH_LETTA_CONFIG_FILE: "/run/config/config.json",
}, },
"env-file": ["/var/lib/letta/runtime.env"], "env-file": ["/var/lib/letta/runtime.env"],
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
"restart-on": ["runtime-config"], "restart-on": ["runtime-config"],
}, },
], ],
@@ -314,11 +359,22 @@ test("consumers on a joined node get their databases from the one foundation sto
await mesh(`assign ${NODE} ${name}`); await mesh(`assign ${NODE} ${name}`);
} }
// The catalogue's manifest as the lab runs it (harness), so the foundation store can be ADOPTED // Load a committed catalog module.json with its container images rewritten to this scenario's
// in place as the one postgres. // pinned digests, so the foundation store can be ADOPTED in place as the one postgres.
function loadManifest(name: string): { manifest: string; broker: boolean } { function loadManifest(name: string): { manifest: string; broker: boolean } {
const manifest = catalogueModule(name, held); const m = JSON.parse(readFileSync(resolve(catalogDir, name, "module.json"), "utf8")) as {
return { manifest, broker: needsBrokerAccount(manifest) }; resources?: { type: string; image?: string; artifact?: string }[];
};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
if (typeof r.image === "string") r.image = pinned(r.image);
else if (typeof r.artifact === "string") {
r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`);
delete r.artifact;
}
}
const manifest = JSON.stringify(m);
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
} }
async function installCatalog(name: string, node: string): Promise<void> { async function installCatalog(name: string, node: string): Promise<void> {
const { manifest, broker } = loadManifest(name); const { manifest, broker } = loadManifest(name);
@@ -360,7 +416,8 @@ test("consumers on a joined node get their databases from the one foundation sto
await mesh(`push anchor`, 600_000); await mesh(`push anchor`, 600_000);
await settled("anchor"); await settled("anchor");
// The consumers ride laptop. // The consumers ride laptop; redis is an ordinary co-located provider (it holds no seat).
await addIssueAssign("redis", redisManifest);
await addIssueAssign("baserow", baserowManifest); await addIssueAssign("baserow", baserowManifest);
await addIssueAssign("letta", lettaManifest); await addIssueAssign("letta", lettaManifest);
await mesh(`push ${NODE}`); await mesh(`push ${NODE}`);
@@ -387,6 +444,7 @@ test("consumers on a joined node get their databases from the one foundation sto
// THE co-residence proof — every module's containers up and stable on the second node. // THE co-residence proof — every module's containers up and stable on the second node.
// ================================================================================================ // ================================================================================================
const expected = [ const expected = [
"redis", "mesh-redis",
"baserow", "mesh-baserow", "baserow", "mesh-baserow",
"letta", "mesh-letta", "letta", "mesh-letta",
]; ];
@@ -462,7 +520,7 @@ test("consumers on a joined node get their databases from the one foundation sto
// reached from laptop over the shared segment) — named for the node that runs it and the module. // reached from laptop over the shared segment) — named for the node that runs it and the module.
// ================================================================================================ // ================================================================================================
const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`); const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`);
for (const acct of ["anchor-postgres", "laptop-baserow", "laptop-letta"]) { for (const acct of ["anchor-postgres", "laptop-redis", "laptop-baserow", "laptop-letta"]) {
assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`); assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`);
} }
@@ -494,26 +552,14 @@ test("consumers on a joined node get their databases from the one foundation sto
assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`); assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`);
} }
// baserow's cache is its own: with no REDIS_HOST the image runs one inside the container, under a // baserow also got its redis-cache binding: the mesh wrote the binding and unsealed the secret,
// password it makes itself, and the mesh grants nothing (novox/hq 081). Three things say so: // and both arrived on the second node (a live redis AUTH is left to the redis single-module bed
// baserow said it chose its own; the cache answers on loopback with the challenge for that password // and the open provider-seal-key work).
// (PONG would be a cache anyone can use, and fails); and nothing was refused a login. const redisBound = await waitForBinding("/var/lib/baserow/redis.json");
const baserowLog = async () => (await on(NODE, `docker logs baserow 2>&1`)).out; assert.equal(redisBound.provision, "redis-cache", `baserow's redis binding is the wrong provision: ${redisBound.provision}`);
let chose = ""; assert.ok(redisBound.as, `baserow's redis binding carries no login:\n${JSON.stringify(redisBound)}`);
let cache = { out: "", ok: false }; const redisSecret = (await must(NODE, `cat /var/lib/baserow/redis.secret`)).trim();
const untilCache = Date.now() + 240_000; assert.ok(redisSecret.length > 0, "baserow's redis secret was not delivered");
while (Date.now() < untilCache) {
chose = await baserowLog();
cache = await on(NODE, `docker exec baserow redis-cli -h 127.0.0.1 ping 2>&1`);
if (/Using embedded baserow redis/.test(chose) && /NOAUTH/.test(cache.out)) break;
await new Promise((r) => setTimeout(r, 5000));
}
assert.match(chose, /Using embedded baserow redis/,
`baserow did not start its own cache:\n${chose.split("\n").filter((l) => /redis/i.test(l)).slice(-15).join("\n")}`);
assert.match(cache.out, /NOAUTH/,
`baserow's own cache does not answer with its password challenge inside the container:\n${cache.out}`);
assert.doesNotMatch(chose, /WRONGPASS|NOPERM/,
`baserow was refused by its cache:\n${chose.split("\n").filter((l) => /WRONGPASS|NOPERM/.test(l)).slice(-15).join("\n")}`);
// Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it. // Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it.
async function waitForBinding(path: string): Promise<{ as: string; provision: string }> { async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {
-38
View File
@@ -259,32 +259,9 @@ test("redis's own password is a secret the vault provides: it authenticates, and
assert.match(issued, /scoped to what it emits and consumes/, issued); assert.match(issued, /scoped to what it emits and consumes/, issued);
await mesh(`assign ${MACHINE} ${name}`); await mesh(`assign ${MACHINE} ${name}`);
} }
// A consumer that needs TWO values from the vault (novox/hq ADR 0094): its `secrets` entry names
// them under local names, and each is a pair of its own. It runs no code — the delivery is what
// is under test. Synthetic, so it wears no catalogue module's name.
const twoSecrets = JSON.stringify({
module: "two-secrets", version: "1", slug: "two",
requires: ["secret"],
secrets: { secret: { first: "/var/lib/two-secrets/first", second: "/var/lib/two-secrets/second" } },
resources: [{ id: "state", type: "directory", path: "/var/lib/two-secrets", mode: "0700" }],
});
await must(`printf %s ${quote(twoSecrets)} > /tmp/two-secrets.json && docker cp /tmp/two-secrets.json mesh-controller:/two-secrets.json`);
await mesh("module add /two-secrets.json");
await mesh(`assign ${MACHINE} two-secrets`);
await mesh(`push ${MACHINE}`); await mesh(`push ${MACHINE}`);
await settled(); await settled();
// Two files, two values, and the vault holds two holders for one module — the identity with the
// local name after it.
const first = (await must(`cat /var/lib/two-secrets/first`)).replace(/\n$/, "");
const second = (await must(`cat /var/lib/two-secrets/second`)).replace(/\n$/, "");
assert.ok(first.length >= 20 && second.length >= 20, "a two-secrets value is empty or implausibly short");
assert.notEqual(first, second, "two local names were given one value");
for (const holderOf of ["mesh_anchor_two_first", "mesh_anchor_two_second"]) {
await until(`the vault holding ${holderOf}`, 90_000, async () =>
(await on(`test -s ${LEDGER}/${holderOf}.json`)).ok ? true : undefined);
}
const running = await must(`docker ps --format '{{.Names}}'`); const running = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-vault", "redis", "mesh-redis"]) { for (const c of ["mesh-vault", "redis", "mesh-redis"]) {
assert.match(running, new RegExp(`(^|\\n)${c}(\\n|$)`), assert.match(running, new RegExp(`(^|\\n)${c}(\\n|$)`),
@@ -358,21 +335,6 @@ test("rotating the secret moves both ends: the new password works, the old one i
}); });
assert.notEqual(after, before); assert.notEqual(after, before);
// Both of the two-secrets consumer's values moved too, apart from each other (ADR 0094).
const firstAfter = await until("the rotated first secret", 180_000, async () => {
const now = (await must(`cat /var/lib/two-secrets/first`)).replace(/\n$/, "");
return now !== "" ? now : undefined;
});
const secondAfter = (await must(`cat /var/lib/two-secrets/second`)).replace(/\n$/, "");
assert.notEqual(firstAfter, secondAfter, "two local names were given one value after rotation");
for (const holderOf of ["mesh_anchor_two_first", "mesh_anchor_two_second"]) {
const h = await until(`the vault recording ${holderOf}'s rotation`, 90_000, async () => {
const got = JSON.parse(await must(`cat ${LEDGER}/${holderOf}.json`)) as Held;
return got.rotations >= 1 ? got : undefined;
});
assert.equal(h.rotations, 1, holderOf);
}
// Three logins. The new one works (redis was restarted on its config — `restart-on`), the old one // Three logins. The new one works (redis was restarted on its config — `restart-on`), the old one
// does not: that third check is what makes it a rotation rather than an addition. // does not: that third check is what makes it a rotation rather than an addition.
await until("redis accepting the rotated password", 180_000, async () => { await until("redis accepting the rotated password", 180_000, async () => {
+8 -82
View File
@@ -41,14 +41,6 @@ const ACME = "/var/lib/acme";
*/ */
const AUTHORITY = "ghcr.io/letsencrypt/pebble:2.5.0"; const AUTHORITY = "ghcr.io/letsencrypt/pebble:2.5.0";
/**
* The second implementation, for the same order (novox/hq 04-ISSUES/020): the certificate
* authority the catalogue itself runs, pinned as the catalogue pins it. If the order, the challenge
* and the handshake agree here as well as against Pebble, the one thing 020 could not rule out — a
* Pebble interop detail — is ruled out; and if they disagree, which side differs is in view.
*/
const SECOND_AUTHORITY = "smallstep/step-ca@sha256:a2b17872915c193259b75a5474c398326f41bd199f0842093e52cf4182bc8270";
let instanceId = ""; let instanceId = "";
function shellQuote(s: string): string { function shellQuote(s: string): string {
@@ -135,18 +127,9 @@ before(async () => {
given: [{ from: "photos", node: "", at: "", values: { name: NAME, port: 8080 } }], given: [{ from: "photos", node: "", at: "", values: { name: NAME, port: 8080 } }],
}))} > ${ACME}/routes.json`, }))} > ${ACME}/routes.json`,
); );
// A real small server, not a netcat loop: the loop's `nc -l -p … -q` is not this machine's netcat, await must(
// so it never listened, and every request through the proxy was refused by the backend — which `nohup sh -c 'while true; do printf "HTTP/1.1 200 OK\\r\\nContent-Length: 5\\r\\n\\r\\nhello" | nc -l -p 8080 -q 1; done' >/dev/null 2>&1 &`,
// read as the certificate never arriving, and hid behind the authority's refusal until the );
// second authority issued one.
await must(`mkdir -p ${ACME}/www && printf hello > ${ACME}/www/index.html`);
await must(`nohup python3 -m http.server 8080 --bind 127.0.0.1 --directory ${ACME}/www >${ACME}/backend.log 2>&1 &`);
let backend = false;
for (let i = 0; i < 20 && !backend; i++) {
({ ok: backend } = await on(`curl -sf http://127.0.0.1:8080/ -o /dev/null`));
if (!backend) await new Promise((r) => setTimeout(r, 1000));
}
assert.ok(backend, `the backend behind the route never answered:\n${(await on(`cat ${ACME}/backend.log`)).out}`);
}, { timeout: 1_200_000 }); }, { timeout: 1_200_000 });
after(async () => { after(async () => {
@@ -194,69 +177,12 @@ test("a public name is served with a certificate the mesh did not issue", {
assert.match(served.out, /hello/); assert.match(served.out, /hello/);
// And it is the authority's certificate, not something self-signed that happens to work. // And it is the authority's certificate, not something self-signed that happens to work.
// The issuer, and the names the certificate is FOR — its alternative names, not its subject: const issuer = await must(
// Pebble, like the public authority it stands in for, leaves the subject empty and puts the
// name in the alternative names alone. The first version of this read the subject and refused
// a valid certificate.
const issued = await must(
`echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` + `echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` +
`| openssl x509 -noout -issuer -ext subjectAltName`, `| openssl x509 -noout -issuer -subject`,
); );
assert.match(issued, /Pebble/i, `the certificate was not issued by the ACME server:\n${issued}`); assert.match(issuer, /Pebble/i, `the certificate was not issued by the ACME server:\n${issuer}`);
assert.match(issued, new RegExp(`DNS:${NAME.replace(".", "\\.")}`), `the certificate is not for the name asked for:\n${issued}`); assert.match(issuer, new RegExp(NAME), `the certificate is not for the name asked for:\n${issuer}`);
});
test("the same order against a second authority: the catalogue's own certificate authority", {
skip, timeout: 900_000,
}, async () => {
// The proxy that served the first test goes; its cache with it, or the certificate Pebble issued
// would be served again and nothing would have been ordered here.
await must(`pkill -f '^/usr/local/bin/mesh-route-proxy' || true; sleep 1; mkdir -p ${ACME}/cache2`);
// The catalogue's authority, as the catalogue runs it: ACME on, listening on its own port, a
// root and an intermediate made at first start. It resolves the name through the machine's
// resolver, which reads the hosts entry the first test wrote.
await must(
`docker run -d --name stepca --network host ` +
`-e DOCKER_STEPCA_INIT_NAME="Lab CA" -e DOCKER_STEPCA_INIT_DNS_NAMES=localhost,127.0.0.1 ` +
`-e DOCKER_STEPCA_INIT_ACME=true -e DOCKER_STEPCA_INIT_REMOTE_MANAGEMENT=false ` +
`-e DOCKER_STEPCA_INIT_PASSWORD=lab-only-password ${SECOND_AUTHORITY}`,
);
let ready = false;
for (let i = 0; i < 90 && !ready; i++) {
({ ok: ready } = await on(`docker exec stepca test -s /home/step/certs/root_ca.crt && curl -sk https://127.0.0.1:9000/health -o /dev/null`));
if (!ready) await new Promise((r) => setTimeout(r, 2000));
}
assert.ok(ready, `the second authority never came up:\n${(await on(`docker logs stepca 2>&1 | tail -30`)).out}`);
await must(`docker exec stepca cat /home/step/certs/root_ca.crt > ${ACME}/stepca-root.pem`);
await must(
`ROUTES=${ACME}/routes.json LISTEN=:80 TLS_LISTEN=:443 ` +
`ACME_CACHE=${ACME}/cache2 ` +
`ACME_DIRECTORY=https://127.0.0.1:9000/acme/acme/directory ` +
`ACME_CA_BUNDLE=${ACME}/stepca-root.pem ` +
`nohup /usr/local/bin/mesh-route-proxy >${ACME}/proxy2.log 2>&1 & sleep 3`,
);
let served = { out: "", ok: false };
for (let i = 0; i < 40 && !served.ok; i++) {
served = await on(`curl -sf --cacert ${ACME}/stepca-root.pem https://${NAME}/ `);
if (!served.ok) await new Promise((r) => setTimeout(r, 2000));
}
if (!served.ok) {
const proxyLog = (await on(`cat ${ACME}/proxy2.log`)).out;
const authority = (await on(`docker logs stepca 2>&1 | tail -40`)).out;
assert.fail(
`the name was never served over TLS from the second authority: ${served.out}\n\n` +
`── the proxy tried:\n${proxyLog}\n── the authority heard:\n${authority}\n`);
}
assert.match(served.out, /hello/);
const issued = await must(
`echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` +
`| openssl x509 -noout -issuer -ext subjectAltName`,
);
assert.match(issued, /Lab CA/, `the certificate was not issued by the second authority:\n${issued}`);
assert.match(issued, new RegExp(`DNS:${NAME.replace(".", "\\.")}`), `the certificate is not for the name asked for:\n${issued}`);
}); });
test("no certificate is ordered for a name the mesh does not route", { test("no certificate is ordered for a name the mesh does not route", {
@@ -267,6 +193,6 @@ test("no certificate is ordered for a name the mesh does not route", {
const { out } = await on( const { out } = await on(
`echo | openssl s_client -connect 127.0.0.1:443 -servername nobody-asked-for-this.example 2>&1 | head -20`, `echo | openssl s_client -connect 127.0.0.1:443 -servername nobody-asked-for-this.example 2>&1 | head -20`,
); );
assert.doesNotMatch(out, /Pebble|Lab CA/i, assert.doesNotMatch(out, /Pebble/i,
`a certificate was obtained for a name nothing routes here:\n${out}`); `a certificate was obtained for a name nothing routes here:\n${out}`);
}); });
-227
View File
@@ -1,227 +0,0 @@
/**
* SPIKE for novox/hq 04-ISSUES/066 — a partly applied declaration leaves a mixed state.
*
* The apply is deliberately not a transaction: every resource is attempted, every failure reported,
* and a failed gate stops what follows it (issue 011, ADR 0053). The question 066 asks is whether a
* pairing exists whose half-state is harmful. This bed makes one, on purpose, and RECORDS what the
* machine is observed doing in it — it is evidence for a decision, not a rule being enforced.
*
* The pair: a config file and a long-lived container that serves the file's content as it was when
* the container started, with a run-once gate between them that validates the file. First push:
* the file says "v1", the gate passes, the service serves v1. Second push: the file says "v2" and
* the gate is made to refuse it. The file is applied before the gate (declaration order), the gate
* fails, the service after it is left as it was — so the machine has v2 on disk and serves v1, and
* reports the push as failed. That is the mixed state. Whether it is harmful is what a person
* decides from this; whether a `together` grouping should exist is what the decision would say.
*
* When such a grouping lands, this bed is where it is proven: the assertions below flip.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "schedule-tick"; // a bare node, as the tick bed uses
const MACHINE = "anchor";
let instanceId = "";
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
/** The control plane, a container on the node. */
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
/**
* How many lines the tick has written so far — tolerant of the run log not existing yet.
*
* The scheduled container appends one line per fire with `date >> /data/runs.log`, and the data
* directory is mounted from /var/lib/schedtest on the machine, so counting newlines there counts
* fires. `wc -l` on an absent file is an error, so a missing file reads as 0 rather than throwing —
* which is exactly the pre-first-fire state.
*/
async function tickLines(): Promise<number> {
const { out } = await on(`wc -l < /var/lib/schedtest/runs.log 2>/dev/null || echo 0`);
const n = Number.parseInt(out.trim(), 10);
return Number.isFinite(n) ? n : 0;
}
async function settled(withinMs = 600_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
// Raise the foundation — store, broker, control — from the bundle.
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
// The node joins its own mesh, so it is a node the mesh can assign to, and start the host so it
// applies what it is pushed AND fires scheduled steps off its clock (the daemon holds one
// Scheduler for the life of the process — novox/hq ADR 0053).
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
function coupled(content: string, gateAccepts: boolean): string {
return JSON.stringify({
module: "coupled", version: "1",
resources: [
{ id: "state", type: "directory", path: "/var/lib/coupled", mode: "0755" },
{ id: "config", type: "file", path: "/var/lib/coupled/config", mode: "0644", content: content + "\n" },
// The gate: validates the file. Made to pass or fail from the manifest, which is the whole
// point — a real validator refusing a real bad config is exactly this shape.
{
id: "validate", type: "container", name: "coupled-validate", image: pinned("alpine"), "run-once": true,
volumes: ["/var/lib/coupled:/data:ro"],
args: ["sh", "-c", gateAccepts ? "test -s /data/config" : "echo 'config refused by the validator' >&2; exit 1"],
},
// The service: reads the file ONCE at start and serves that for its life, the way most
// servers read their configuration.
{
id: "service", type: "container", name: "coupled-service", image: pinned("alpine"), network: "host",
volumes: ["/var/lib/coupled:/data:ro"],
args: ["sh", "-c", "v=$(cat /data/config); while true; do printf 'HTTP/1.1 200 OK\\r\\nContent-Length: %s\\r\\n\\r\\n%s' \"${#v}\" \"$v\" | nc -l -p 8099; done"],
"restart-on": ["config"],
},
],
});
}
async function served(): Promise<string> {
return (await on(`curl -s --max-time 3 http://127.0.0.1:8099/ || true`)).out.trim();
}
test("a coupled pair half-applied: the file moved, the gate refused, the service serves the old file — the machine is observed in the mixed state", {
skip, timeout: 900_000,
}, async () => {
await must(`printf %s ${quote(coupled("v1", true))} > /tmp/coupled.json && docker cp /tmp/coupled.json mesh-controller:/coupled.json`);
await mesh("module add /coupled.json");
await mesh(`assign ${MACHINE} coupled`);
await mesh(`push ${MACHINE}`);
await settled();
let first = "";
for (let i = 0; i < 20 && first !== "v1"; i++) {
first = await served();
if (first !== "v1") await new Promise((r) => setTimeout(r, 2000));
}
assert.equal(first, "v1", "the service does not serve the first config");
// The change: a new file the validator refuses. Registered again under the same name so the
// mesh sends a new declaration; the file is applied, the gate fails, the service stays.
await must(`printf %s ${quote(coupled("v2", false))} > /tmp/coupled.json && docker cp /tmp/coupled.json mesh-controller:/coupled.json`);
await mesh("module add /coupled.json");
const pushed = await on(`docker exec mesh-controller /mesh-controller push ${MACHINE}`);
// The push is sent; what the machine did with it is read from its report.
let report = "";
for (let i = 0; i < 30; i++) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
report = asked.out;
if (/"outcome":\s*"failed"/.test(report)) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(report, /"outcome":\s*"failed"/, `the machine did not report a failed apply:\n${pushed.out}\n${report}`);
// THE OBSERVATION. The file on disk is the new one; the service still serves the old one.
const onDisk = (await must(`cat /var/lib/coupled/config`)).trim();
const answered = await served();
assert.equal(onDisk, "v2", "the file was not applied before the gate");
assert.equal(answered, "v1", `the service was restarted onto a config the validator refused: ${answered}`);
console.log(`OBSERVED: config on disk = ${onDisk}, service serves = ${answered}, report = failed — the mixed state of novox/hq 04-ISSUES/066`);
});
-1
View File
@@ -93,7 +93,6 @@ before(async () => {
step: "getting the machine ready to be bootstrapped — the installer never ran", step: "getting the machine ready to be bootstrapped — the installer never ran",
why: (err as Error).message, why: (err as Error).message,
report: [], report: [],
said: "",
}; };
} }
console.log(result.report.join("\n")); console.log(result.report.join("\n"));
+7 -35
View File
@@ -26,8 +26,6 @@ export interface GenesisResult {
step: string; step: string;
why: string; why: string;
report: string[]; report: string[];
/** Everything the installer printed on its last attempt — what a bed asserts a refusal names. */
said: string;
} }
export interface GenesisOptions { export interface GenesisOptions {
@@ -82,23 +80,6 @@ export interface GenesisOptions {
hostBinary?: string; hostBinary?: string;
/** What of the catalogue to build. A branch under test is the usual reason this is not main. */ /** What of the catalogue to build. A branch under test is the usual reason this is not main. */
catalogRef?: string; catalogRef?: string;
/**
* Raise the machine adopted (novox/hq ADR 0100): what it runs and its firewall are kept. Without
* it the installer raises a converged node, and refuses a machine in use.
*/
adopted?: boolean;
/** Further installer flags, as the operator would type them — `--registry-port 5100`, `--dry-run`. */
flags?: string[];
/**
* How many times to run the installer. Three by default, for a pull the internet rate-limited; a
* bed that expects a REFUSAL runs it once, because a refusal is the answer, not a flake.
*/
attempts?: number;
/**
* Whether to ask the machine if it became a working mesh of one afterwards. Off for a run that is
* not meant to raise one — a dry run, or a refusal the bed expects.
*/
verify?: boolean;
log?: (m: string) => void; log?: (m: string) => void;
} }
@@ -117,10 +98,9 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
const log = o.log ?? (() => {}); const log = o.log ?? (() => {});
const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`]; const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`];
let said = "";
const stop = (step: string, why: string): GenesisResult => { const stop = (step: string, why: string): GenesisResult => {
report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`); report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
return { ok: false, step, why, report, said }; return { ok: false, step, why, report };
}; };
const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => { const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => {
@@ -156,9 +136,7 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
// the lab stands in for that by copying the whole tree once. // the lab stands in for that by copying the whole tree once.
const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`); const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`);
execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]); execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]);
// Cleared first: a bed that runs genesis more than once (a refusal, then the raise) finds the last await must(`mkdir -p ${catalogueOnMachine}/modules`);
// run's staging files, and the machine refuses to open them for the push.
await must(`rm -f /tmp/catalogue.tar /tmp/foundation-template.lock && mkdir -p ${catalogueOnMachine}/modules`);
await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar"); await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar");
await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`); await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`);
// The three genesis itself needs must be present, or the pivot cannot even begin — checked here // The three genesis itself needs must be present, or the pivot cannot even begin — checked here
@@ -206,8 +184,6 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
`--private-network wireguard`, `--private-network wireguard`,
`--packet-filter nftables`, `--packet-filter nftables`,
`--host ${HOST_PATH}`, `--host ${HOST_PATH}`,
...(o.adopted ? [`--adopted`] : []),
...(o.flags ?? []),
// A service when the packaging was installed above (survives a reboot); otherwise the // A service when the packaging was installed above (survives a reboot); otherwise the
// background process, which does not — the installer refuses to invent a unit either way. // background process, which does not — the installer refuses to invent a unit either way.
...(o.hostService ? [] as string[] : [`--host-in-background`]), ...(o.hostService ? [] as string[] : [`--host-in-background`]),
@@ -217,24 +193,20 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
// but because the installer is idempotent by design and says so, and because the one thing that // but because the installer is idempotent by design and says so, and because the one thing that
// fails for a reason which goes away by itself is a pull: the store, broker and registry come // fails for a reason which goes away by itself is a pull: the store, broker and registry come
// from the internet, and a rate-limited anonymous pull is not this mesh's fault. // from the internet, and a rate-limited anonymous pull is not this mesh's fault.
let said = "";
let step = ""; let step = "";
const attempts = o.attempts ?? 3; for (let attempt = 1; attempt <= 3; attempt++) {
for (let attempt = 1; attempt <= attempts; attempt++) {
const ran = await on(command, 2_400_000); const ran = await on(command, 2_400_000);
said = ran.out; said = ran.out;
log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`); log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`);
if (ran.ok) { step = ""; break; } if (ran.ok) { step = ""; break; }
step = stepIn(said) || "an unnamed step"; step = stepIn(said);
if (attempt < attempts) { if (attempt < 3) {
log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`); log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`);
await new Promise((r) => setTimeout(r, 30_000)); await new Promise((r) => setTimeout(r, 30_000));
} }
} }
if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n")); if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));
if (o.verify === false) {
report.push(`\nThe installer finished; not asked whether it raised a mesh (verify: false).`);
return { ok: true, step: "", why: "", report, said };
}
// ------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------
// Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting // Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting
@@ -314,5 +286,5 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
} }
report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`); report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`);
return { ok: true, step: "", why: "", report, said }; return { ok: true, step: "", why: "", report };
} }
+6 -117
View File
@@ -40,14 +40,6 @@ const UPSTREAM_STORE =
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"; "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
const UPSTREAM_BROKER = const UPSTREAM_BROKER =
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"; "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
/**
* And the bus, for `foundation-first-node-nats.lock` — the bundle the mesh raises since it stopped
* speaking AMQP (novox/hq ADR 0131). The digest is the bundle's own: what the registry served was a
* copy of the upstream image, so the same digest resolves on Docker Hub, and this is a prefix being
* removed rather than a reference being replaced.
*/
const UPSTREAM_BUS =
"nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927";
/** /**
* The foundation bundle as a machine should receive it. * The foundation bundle as a machine should receive it.
@@ -61,7 +53,6 @@ export function foundationBundle(path: string, held: HeldImage[]): string {
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE); text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
text = text.replaceAll( text = text.replaceAll(
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER); /[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
text = text.replaceAll(/[A-Za-z0-9_.:-]+:[0-9]+\/nats@sha256:[0-9a-f]{64}/g, UPSTREAM_BUS);
return pinnedInto(text, held); return pinnedInto(text, held);
} }
@@ -246,116 +237,14 @@ export async function assertUniversalInvariants(
/** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */ /** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */
export const FILTER_MODULE = "nftables"; export const FILTER_MODULE = "nftables";
// --- the catalogue: a bed installs a module by reading its manifest, never by carrying a copy ---- /** The catalogue's manifest for a module, under either spelling of MESH_LAB_CATALOG: the modules
* directory, or the checkout that holds it. */
/**
* The catalogue's `modules/` directory: MESH_LAB_CATALOG under either spelling (the checkout, or
* its modules directory). Named, or absent — never guessed from a sibling path: the receipt claims
* the catalogue the run was pointed at (src/repos.ts), and a catalogue read from somewhere the
* receipt does not name is the drift this exists to close.
*
* Beds used to build the manifests they install inline, as literals copied from the catalogue when
* each bed was written. The copies did not move when the catalogue did, so a catalogue change was
* proven nowhere — and a bed that installs a copy proves the copy (novox/hq 04-ISSUES/073). A bed
* reads the catalogue, or it does not install a catalogue module; `beds-read-the-catalogue.test.ts`
* refuses an inline copy that names one.
*/
export function catalogueDir(): string {
const named = process.env["MESH_LAB_CATALOG"];
if (!named) throw new Error("MESH_LAB_CATALOG is not set to a checkout of mesh-catalog (or its modules directory)");
const candidates = [resolve(named, "modules"), resolve(named)];
for (const dir of candidates) {
// Known by the registry's manifest, which genesis reads from the catalogue and always will —
// not the control plane's, which lives in the control plane's own repository (ADR 0069).
if (existsSync(resolve(dir, "distribution", "module.json"))) return dir;
}
throw new Error(`no catalogue: MESH_LAB_CATALOG=${named} and no distribution/module.json under ${candidates.join(" or ")}`);
}
/** Whether a catalogue is where a bed will look — for a skip guard, which says so instead of failing. */
export function catalogueIsPresent(): string | false {
try { catalogueDir(); return false; } catch (err) { return (err as Error).message; }
}
/** The catalogue's manifest for a module, as a path. */
export function catalogueManifest(module: string): string { export function catalogueManifest(module: string): string {
const path = resolve(catalogueDir(), module, "module.json"); const dir = process.env["MESH_LAB_CATALOG"] ?? "";
if (!existsSync(path)) throw new Error(`no manifest for ${module} at ${path}`); for (const candidate of [resolve(dir, module, "module.json"), resolve(dir, "modules", module, "module.json")]) {
return path; if (existsSync(candidate)) return candidate;
}
/** What the lab may rewrite in a catalogue manifest, and nothing else. */
export interface ForTheLab {
/**
* The image repository each build artifact was built as on this workstation, by artifact name.
* A module's own runtime is `mesh-runtime-<module>` by default — what `scripts/build-module-runtime.sh`
* tags and what the scenarios stock; a bed names it only where the scenario stocks another name.
* An artifact this does not name is refused: the bed must say what stands in for the builder.
*/
artifacts?: Record<string, string>;
/**
* Host-port remaps, where one machine carries modules whose published ports collide —
* `{ "8080": "8090:8080" }`, applied to every container of the module. The container side never
* changes.
*/
ports?: Record<string, string> | undefined;
/**
* Environment a container gets in the lab that it does not get in the mesh — an address the bed
* stands up in place of a real upstream, and nothing else. Merged over the manifest's own.
*/
env?: Record<string, Record<string, string>>;
}
/**
* A catalogue manifest as a machine in the lab can run it: the mesh's build section gone (the lab
* stocks images rather than building), each artifact replaced by the image the machine holds for it,
* every image pinned to what the machine holds or the upstream digest the catalogue pins, and the
* declared lab rewrites applied. Everything else is the catalogue's, verbatim — which is the point.
*/
export function catalogueModule(module: string, held: HeldImage[], lab: ForTheLab = {}): string {
const m = JSON.parse(readFileSync(catalogueManifest(module), "utf8")) as {
resources?: { id: string; type: string; image?: string; artifact?: string; ports?: string[]; env?: Record<string, string> }[];
build?: { artifacts?: { name: string; kind: string; from?: string }[] };
};
const artifacts: Record<string, string> = { runtime: `mesh-runtime-${module}`, ...(lab.artifacts ?? {}) };
// An upstream artifact is somebody else's image, which the mesh's builder copies into its own
// registry (ADR 0096). The lab stands in for the builder by using the reference the manifest
// pins, which the machine pulls over its uplink — the same bytes, without the copy.
const upstream = new Map<string, string>();
for (const a of m.build?.artifacts ?? []) {
if (a.kind === "upstream" && a.from) upstream.set(a.name, a.from);
} }
for (const r of m.resources ?? []) { throw new Error(`no manifest for ${module} under MESH_LAB_CATALOG=${dir || "(unset)"}`);
if (r.type !== "container") continue;
if (typeof r.artifact === "string" && upstream.has(r.artifact) && !lab.artifacts?.[r.artifact]) {
r.image = onTheMachine(upstream.get(r.artifact)!, held);
delete r.artifact;
} else if (typeof r.artifact === "string") {
const repository = artifacts[r.artifact];
assert.ok(repository,
`${module}'s container '${r.id}' names the "${r.artifact}" artifact, which the mesh would ` +
`build. The lab does not build: the bed must say which stocked image stands in for it ` +
`(artifacts: { ${r.artifact}: "<repository>" }).`);
const reference = referenceFor(held, repository);
assert.ok(reference,
`${module}'s "${r.artifact}" artifact is ${repository} and this scenario stocked no such ` +
`image. Add it to the scenario's images: and build it (scripts/build-module-runtime.sh ${module}).`);
r.image = reference;
delete r.artifact;
} else if (typeof r.image === "string") {
r.image = onTheMachine(r.image, held);
}
if (lab.ports && Array.isArray(r.ports)) r.ports = r.ports.map((p) => lab.ports![p] ?? p);
const env = lab.env?.[r.id];
if (env) r.env = { ...(r.env ?? {}), ...env };
}
delete m.build;
return JSON.stringify(m);
}
/** Whether a manifest's runtime dials the broker — the module then needs a scoped broker account. */
export function needsBrokerAccount(manifest: string): boolean {
return manifest.includes("MESH_BROKER_FILE");
} }
function shellQuote(s: string): string { function shellQuote(s: string): string {
+44 -11
View File
@@ -26,7 +26,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -39,7 +39,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "local-model-bed"; const SCENARIO = "local-model-bed";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -119,8 +119,7 @@ async function settled(withinMs = 600_000): Promise<void> {
async function addAssign(name: string, manifest: string): Promise<void> { async function addAssign(name: string, manifest: string): Promise<void> {
await must(`printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); await must(`printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`); await mesh(`module add /${name}.json`);
// No `module issue`: neither module speaks on the bus, and issuing a module with no broker await mesh(`module issue ${name} --node ${MACHINE}`);
// secret to deliver into is refused (novox/hq issue 078).
await mesh(`assign ${MACHINE} ${name}`); await mesh(`assign ${MACHINE} ${name}`);
} }
@@ -154,13 +153,47 @@ after(async () => {
test("a node hosting a model answers model-access, and the consumer is handed its endpoint", { test("a node hosting a model answers model-access, and the consumer is handed its endpoint", {
skip, timeout: 1_500_000, skip, timeout: 1_500_000,
}, async () => { }, async () => {
// Both manifests are the catalogue's own (novox/hq 04-ISSUES/073). The provider: ollama runs the const ollamaImage = pinned("ollama/ollama");
// model server and `provides: ["model-access"]` at node scope, serving its port and model. It mints
// nothing — provides/serves are declaration the mesh reads, so there is no runtime container, only // The provider: ollama runs the model server and `provides: ["model-access"]` at node scope, serving
// the server. The consumer requires model-access and is answered by the local node: no secret (the // its port and model. It mints nothing — provides/serves are declaration the mesh reads, so there is
// local server is keyless), only the bound endpoint facts, templated into an openai.env the mesh writes. // no runtime container, only the server.
const ollamaManifest = catalogueModule("ollama", held); const ollamaManifest = JSON.stringify({
const consumerManifest = catalogueModule("local-model-consumer", held); module: "ollama",
version: "1",
capabilities: ["container-runtime"],
provides: [{ name: "model-access", scope: "node" }],
listens: [{ port: 11434, protocol: "tcp", from: "machine", why: "local consumers reaching the model server" }],
serves: { "model-access": { port: 11434, model: "llama3.2" } },
resources: [
{ id: "state", type: "directory", path: "/services/ollama", mode: "0700" },
{
id: "server", type: "container", name: "ollama",
image: ollamaImage, network: "host", env: { OLLAMA_HOST: "0.0.0.0:11434" },
volumes: ["/services/ollama:/root/.ollama"],
},
],
});
// The consumer: it requires model-access and is answered by the local node. No secret (the local
// server is keyless), only the bound endpoint facts, templated into an openai.env the mesh writes.
const consumerManifest = JSON.stringify({
module: "local-model-consumer",
version: "1",
slug: "local",
requires: ["model-access"],
binds: { "model-access": "/var/lib/local-model-consumer/model.json" },
resources: [
{ id: "state", type: "directory", path: "/var/lib/local-model-consumer", mode: "0700" },
{ id: "config", type: "directory", path: "/var/lib/local-model-consumer/config", mode: "0700" },
{
id: "openai-env", type: "file", path: "/var/lib/local-model-consumer/config/openai.env", mode: "0600",
content:
"OPENAI_BASE_URL=http://${bound:model-access:at}:${bound:model-access:port}/v1\n" +
"OPENAI_MODEL=${bound:model-access:model}\nOPENAI_API_KEY=local\n",
},
],
});
await addAssign("ollama", ollamaManifest); await addAssign("ollama", ollamaManifest);
await addAssign("local-model-consumer", consumerManifest); await addAssign("local-model-consumer", consumerManifest);
+47 -35
View File
@@ -23,7 +23,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -36,7 +36,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "redis-node"; const SCENARIO = "redis-node";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -139,12 +139,51 @@ after(async () => {
test("the mesh grants a consumer redis's cache, and the credential it delivers authenticates", { test("the mesh grants a consumer redis's cache, and the credential it delivers authenticates", {
skip, timeout: 900_000, skip, timeout: 900_000,
}, async () => { }, async () => {
// The PROVIDER: the catalogue's redis (novox/hq 04-ISSUES/074) — server and a broker-bound // The PROVIDER: redis in its committed shape — server and a broker-bound runtime on the private
// runtime on the private redis network, the runtime running the provisioner. It requires a // redis network, the runtime running the provisioner.
// `secret` for its own password, so the vault that provides one is installed beside it, exactly const redisManifest = JSON.stringify({
// as the vault bed does. module: "redis",
const vaultManifest = catalogueModule("mesh-vault", held); version: "1",
const redisManifest = catalogueModule("redis", held); provides: [{ name: "redis-cache", scope: "mesh" }],
serves: { "redis-cache": {} },
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
grants: { "redis-cache": "/var/lib/redis-module/grants" },
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
{
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
},
{ id: "net", type: "network", name: "redis" },
{
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "redis",
ports: ["6379"],
volumes: ["/services/redis/data:/data", "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro"],
args: ["/etc/redis/redis.conf"],
},
{
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
network: "redis",
volumes: [
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
"/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro",
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
MESH_PROVISION_REDIS: "redis:6379",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
},
},
],
});
// The CONSUMER: a module that requires redis-cache and no more. It runs no code here — the mesh // The CONSUMER: a module that requires redis-cache and no more. It runs no code here — the mesh
// delivers it a bound file (where redis is, and the login to present) and its sealed password, // delivers it a bound file (where redis is, and the login to present) and its sealed password,
@@ -152,9 +191,6 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
const consumerManifest = JSON.stringify({ const consumerManifest = JSON.stringify({
module: "cacheuser", module: "cacheuser",
version: "1", version: "1",
// `mesh_anchor_cacheuser` is 21 characters, over the 20 a backend keeps (ADR 0049); the slug
// makes the consumer identity `mesh_anchor_cache`.
slug: "cache",
requires: ["redis-cache"], requires: ["redis-cache"],
// `contributes` (not just `requires`) is what makes a consumer *ask* — the grant forms from a // `contributes` (not just `requires`) is what makes a consumer *ask* — the grant forms from a
// contribution. It must be non-empty; redis's provisioner ignores the value (it uses the login // contribution. It must be non-empty; redis's provisioner ignores the value (it uses the login
@@ -165,10 +201,6 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
resources: [{ id: "state", type: "directory", path: "/var/lib/cacheuser", mode: "0700" }], resources: [{ id: "state", type: "directory", path: "/var/lib/cacheuser", mode: "0700" }],
}); });
await must(`printf %s ${quote(vaultManifest)} > /tmp/mesh-vault.json && docker cp /tmp/mesh-vault.json mesh-controller:/mesh-vault.json`);
await mesh("module add /mesh-vault.json");
await mesh(`module issue mesh-vault --node ${MACHINE}`);
await mesh(`assign ${MACHINE} mesh-vault`);
await must(`printf %s ${quote(redisManifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`); await must(`printf %s ${quote(redisManifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
await mesh("module add /redis.json"); await mesh("module add /redis.json");
await mesh(`module issue redis --node ${MACHINE}`); await mesh(`module issue redis --node ${MACHINE}`);
@@ -220,24 +252,4 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
assert.doesNotMatch(authed.out, /WRONGPASS|NOPERM|no password/i, assert.doesNotMatch(authed.out, /WRONGPASS|NOPERM|no password/i,
`the consumer's mesh-delivered credential did not authenticate — the two ends do not agree:\n${authed.out}`); `the consumer's mesh-delivered credential did not authenticate — the two ends do not agree:\n${authed.out}`);
assert.match(authed.out, /PONG/, `expected PONG authenticating as the granted consumer:\n${authed.out}`); assert.match(authed.out, /PONG/, `expected PONG authenticating as the granted consumer:\n${authed.out}`);
// And the provider needed no seal key to do it: the password reached it as a file the host left
// after unsealing, so MESH_SEAL_KEY is set nowhere (novox/hq ADR 0048). Carried over from the
// retired provider-uses-mesh-credential bed, whose other proofs this bed makes with the mesh
// writing the contributions rather than the bed.
const runtimeEnv = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`);
assert.doesNotMatch(runtimeEnv, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${runtimeEnv}`);
// A grant means exactly the consumer's own keys — `<login>:*`, the keyspace redis's provisioner
// scopes the ACL user to: under it the consumer reads and writes, outside it and on the server as
// a whole it is refused. Carried over from the large mesh bed's retired cache-grant test —
// without this a provisioner that granted everything would keep every bed green.
const asConsumer = (command: string) =>
on(`docker exec redis redis-cli --user ${quote(as)} --pass ${quote(password)} --no-auth-warning ${command} 2>&1`);
assert.match((await asConsumer(`SET ${as}:proof yes`)).out, /OK/, "the consumer cannot write under its own login");
assert.match((await asConsumer(`GET ${as}:proof`)).out, /yes/, "the consumer cannot read back what it wrote");
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
"the consumer wrote outside its own keys, so the grant means more than it says");
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
"the consumer flushed the whole server, so the grant means more than it says");
}); });
+531 -107
View File
@@ -18,13 +18,13 @@
import { test, before, after } from "node:test"; import { test, before, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync } from "node:fs"; import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts"; import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import type { HeldImage } from "../../src/pinning.ts"; import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import { incus } from "../../src/incus/client.ts"; import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts"; import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts"; import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
@@ -37,6 +37,7 @@ const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const builder = process.env["MESH_LAB_BUILDER"] ?? ""; const builder = process.env["MESH_LAB_BUILDER"] ?? "";
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */ /** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */
const moduleExamples = process.env["MESH_LAB_MODULES"] ?? "";
const skip = !capability.usable const skip = !capability.usable
? `lab not usable: ${capability.why}` ? `lab not usable: ${capability.why}`
@@ -44,8 +45,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
// The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074). : false;
: catalogueIsPresent() || false;
const SCENARIO = "two-nodes"; const SCENARIO = "two-nodes";
let instanceId = ""; let instanceId = "";
@@ -200,24 +200,6 @@ function tokenFrom(said: string): string {
return found; return found;
} }
/** The builder started by hand on the anchor, against the foundation broker's plain port on
* loopback — the one that builds until a builder module can (see the retired test's note). */
async function startBuilder(): Promise<void> {
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
// return, so it is pushed whenever the machine has none.
if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) {
await incus([
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
"--mode", "0755",
], 180_000);
}
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
`(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
}
before(async () => { before(async () => {
if (skip) return; if (skip) return;
@@ -250,8 +232,6 @@ before(async () => {
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`); const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
assert.equal(running.out.trim(), "yes", assert.equal(running.out.trim(), "yes",
"the host did not come back after a restore, so nothing would apply anything"); "the host did not come back after a restore, so nothing would apply anything");
// The hand-started builder is memory too, and the snapshot is disk.
if (builder) await startBuilder();
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` + console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
`and started the host again`); `and started the host again`);
@@ -278,7 +258,17 @@ before(async () => {
// A build machine, so anything here can ask the mesh to build something. Placed rather than // A build machine, so anything here can ask the mesh to build something. Placed rather than
// assumed: nothing else in this scenario would start one. // assumed: nothing else in this scenario would start one.
if (builder) await startBuilder(); if (builder) {
await incus([
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
"--mode", "0755",
], 180_000);
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
await must("anchor",
`MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
}
if (warming) { if (warming) {
// Snapshotted only now, with everything up: a state worth returning to is the one after the // Snapshotted only now, with everything up: a state worth returning to is the one after the
// part nobody wants to repeat. // part nobody wants to repeat.
@@ -328,7 +318,7 @@ test("both machines join it, and the token is all they need", { skip, timeout: 9
test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => { test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
// The whole argument, on real machines: the two ends must hold the SAME password, and it must // The whole argument, on real machines: the two ends must hold the SAME password, and it must
// appear nowhere the mesh or the broker could read it. // appear nowhere the mesh or the broker could read it.
await must("anchor", `printf %s '{"module":"a-store","version":"1",` + await must("anchor", `printf %s '{"module":"postgres","version":"1",` +
`"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` + `"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` +
`"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` + `"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` +
`"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`); `"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
@@ -336,8 +326,7 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
// real program takes a credential: a sealed file is a password alone, and almost nothing reads // real program takes a credential: a sealed file is a password alone, and almost nothing reads
// one. The mesh cannot compose the document — it discarded the value — so the module supplies it // one. The mesh cannot compose the document — it discarded the value — so the module supplies it
// with `${secret:...}` in it and the host, the only thing that sees both halves, fills it in. // with `${secret:...}` in it and the host, the only thing that sees both halves, fills it in.
// A slug, so its identity on a backend stays within an S3 access key's 20 characters (ADR 0049). await must("anchor", `printf %s '{"module":"meshboard","version":"1",` +
await must("anchor", `printf %s '{"module":"meshboard","version":"1","slug":"board",` +
`"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` + `"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` +
`"binds":{"postgres-database":"/etc/meshboard/database.json"},` + `"binds":{"postgres-database":"/etc/meshboard/database.json"},` +
`"secrets":{"postgres-database":"/etc/meshboard/database.password"},` + `"secrets":{"postgres-database":"/etc/meshboard/database.password"},` +
@@ -353,20 +342,14 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
await mesh("overlay place laptop --site lab"); await mesh("overlay place laptop --site lab");
for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`); for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
await mesh("assign anchor a-store"); await mesh("assign anchor postgres");
await mesh("assign laptop meshboard"); await mesh("assign laptop meshboard");
for (const machine of ["anchor", "laptop"]) { for (const machine of ["anchor", "laptop"]) {
// Once. On a warm return the host is already running (see `before`); a second one would await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
// consume the same queue and apply the same declaration twice, concurrently.
await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
} }
await mesh("push"); await mesh("push");
await new Promise((r) => setTimeout(r, 8000)); await new Promise((r) => setTimeout(r, 8000));
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
// and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
// keeps the hub closed and nothing on the laptop reaches anchor over the private network.
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim(); const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
// Named after the machine *and* the module, because a consumer is both (novox/hq // Named after the machine *and* the module, because a consumer is both (novox/hq
@@ -388,7 +371,7 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
assert.match(filled, /^PGPASSWORD=.+$/m, `the password was never put in:\n${filled}`); assert.match(filled, /^PGPASSWORD=.+$/m, `the password was never put in:\n${filled}`);
assert.ok(filled.includes(`PGPASSWORD=${onConsumer}`), assert.ok(filled.includes(`PGPASSWORD=${onConsumer}`),
`the file holds a different password from the credential file:\n${filled}`); `the file holds a different password from the credential file:\n${filled}`);
assert.match(filled, /^PGUSER=mesh_laptop_board$/m, assert.match(filled, /^PGUSER=mesh_laptop_meshboard$/m,
`the consumer was not told what name to present:\n${filled}`); `the consumer was not told what name to present:\n${filled}`);
assert.match(filled, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${filled}`); assert.match(filled, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${filled}`);
assert.doesNotMatch(filled, /\$\{/, assert.doesNotMatch(filled, /\$\{/,
@@ -634,7 +617,6 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
`"capabilities":["container-runtime"],` + `"capabilities":["container-runtime"],` +
`"claims":[{"name":"the-artifact-store","scope":"node"}],` + `"claims":[{"name":"the-artifact-store","scope":"node"}],` +
`"serves":{"artifact-store":{"port":5000}},` + `"serves":{"artifact-store":{"port":5000}},` +
`"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` +
`"resources":[` + `"resources":[` +
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` + `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` + `{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
@@ -648,15 +630,10 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
await mesh("module add /registry.json"); await mesh("module add /registry.json");
await mesh("assign anchor registry"); await mesh("assign anchor registry");
await mesh("push anchor"); await mesh("push anchor");
// The store's image is pulled from upstream at apply, over the uplink; that takes what it takes. await new Promise((r) => setTimeout(r, 12_000));
let names = "";
for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) {
await new Promise((r) => setTimeout(r, 3000));
names = await must("anchor", `docker ps --format '{{.Names}}'`);
}
// Running, and answering — a container that is up is not a registry that replies. // Running, and answering — a container that is up is not a registry that replies.
assert.match(names, /mesh-registry/, assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
`the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`);
let answers = false; let answers = false;
for (let i = 0; i < 20 && !answers; i++) { for (let i = 0; i < 20 && !answers; i++) {
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok; answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
@@ -677,11 +654,8 @@ test("a machine serves its internal name with a certificate the mesh issued", {
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity). // The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
// Asserted with a real handshake: a certificate that parses and does not chain fails at the // Asserted with a real handshake: a certificate that parses and does not chain fails at the
// moment something connects, which is the worst place to find out. // moment something connects, which is the worst place to find out.
// The port the handshake below is tried on, declared: the anchor filters what its modules
// did not declare (ADR 0088), and a test server on an undeclared port proves only that.
await must("anchor", `printf %s '{"module":"served","version":"1",` + await must("anchor", `printf %s '{"module":"served","version":"1",` +
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` + `"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
`"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` +
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` + `"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
`> /tmp/served.json`); `> /tmp/served.json`);
await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`); await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
@@ -840,11 +814,117 @@ test("a machine filters exactly what its modules declared, and nothing else", {
"the port stayed open after the module that wanted it was removed"); "the port stayed open after the module that wanted it was removed");
}); });
// The builder as a module the mesh assigns, with a credential the mesh delivered, is what genesis test("the builder is a module the mesh assigns, with a credential the mesh delivered", {
// proves now (genesis-single installs the catalogue's builder through the installer, novox/hq ADR skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
// 0069). The test that lived here declared the builder's image as an upstream artifact by the bare timeout: 900_000,
// image ID the lab holds, which is not a reference a registry copy can fetch (ADR 0096); retired }, async () => {
// 2026-09-21 rather than rewritten into a second genesis. // Until this, the builder was a program somebody started on a machine with whatever credential
// they had to hand — in practice the broker's administrative one. A program documented as
// holding its own credential and given somebody else's is worse than one with no story at all.
//
// So: the mesh issues a scoped account, seals it to the machine, and delivers it with the
// declaration. Nobody types it and the mesh cannot read it back.
await must("anchor", `mkdir -p /root/builder && printf %s '{"module":"builder","version":"1",` +
`"requires":["artifact-store"],"capabilities":["container-runtime"],` +
`"claims":[{"name":"the-build-machine","scope":"node"}],` +
`"binds":{"artifact-store":"/var/lib/mesh/builder/artifact-store.json"},` +
`"own-secrets":{"broker":"/var/lib/mesh/builder/broker"},` +
`"build":{"artifacts":[{"name":"builder","kind":"upstream",` +
`"from":"${pinned("mesh-builder")}"}]},` +
`"resources":[` +
`{"id":"state","type":"directory","path":"/var/lib/mesh/builder","mode":"0700"},` +
`{"id":"workspace","type":"directory","path":"/var/lib/mesh/builder/workspace","mode":"0700"},` +
`{"id":"run","type":"container","name":"mesh-builder","artifact":"builder",` +
`"network":"host",` +
`"volumes":["/var/lib/mesh/builder:/var/lib/mesh/builder",` +
`"/var/run/docker.sock:/var/run/docker.sock"],` +
`"env":{"MESH_BROKER_FILE":"/var/lib/mesh/builder/broker",` +
`"MESH_BINDING":"/var/lib/mesh/builder/artifact-store.json",` +
`"MESH_WORKSPACE":"/var/lib/mesh/builder/workspace"}}]}' > /root/builder/module.json`);
await must("anchor", `cd /root/builder && git init -q . && git add -A && ` +
`git -c user.email=lab -c user.name=lab commit -qm builder`);
// The builder's own image is built by the builder that is already running — the same
// chicken-and-egg as the registry, resolved the same way. The one started by hand does this
// last piece of work and is then replaced by the module it just built.
await mesh("build /root/builder --wait 300s", 420_000);
// The mesh makes the account and seals the URL to this machine. Nothing is printed that would
// work if it were pasted somewhere else.
const issued = await mesh("builder issue lab-builder --node anchor");
assert.match(issued, /sealed to anchor/, issued);
assert.doesNotMatch(issued, /amqps:\/\/lab-builder:/,
"the credential was printed, so the one copy that matters is on a terminal");
// Now the hand-started one goes, or two builders race for the same queue and whichever answers
// proves nothing. By process name: `pkill -f` matches the shell running it too, which kills the
// connection carrying the command and hangs the caller waiting for a reply that will never
// come. Cost an hour once, in this file.
await on("anchor", `pkill -x mesh-builder`);
await new Promise((r) => setTimeout(r, 2000));
assert.ok(!(await on("anchor", `pgrep -x mesh-builder`)).ok,
"the hand-started builder is still running, so this would test that one");
await mesh("assign anchor builder");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 20_000));
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
assert.match(running, /mesh-builder/,
`the builder was assigned and is not running:\n${running}\n` +
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
// Running is not connected. A builder that cannot reach the broker sits there, and every
// outward sign — the container is up, the credential is on disk — says it is working.
await new Promise((r) => setTimeout(r, 5000));
const said = await on("anchor", `docker logs mesh-builder 2>&1 | tail -20`);
assert.doesNotMatch(said.out, /cannot reach the broker/,
`the builder is running and cannot reach the broker:\n${said.out}`);
// The credential arrived, is readable only by the machine, and is the scoped account rather
// than the broker's own.
assert.match(await must("anchor", `stat -c %a /var/lib/mesh/builder/broker`), /^600/);
const credential = await must("anchor", `cat /var/lib/mesh/builder/broker`);
assert.match(credential, /"url":"amqps:\/\/lab-builder:/,
"the builder is using an account that is not its own");
assert.doesNotMatch(credential, /guest:guest/, "the builder holds the broker's own account");
// And what to check the broker against. A mesh's broker presents a certificate of the mesh's
// own, so a URL alone reaches only a broker some public authority vouches for — which is no
// mesh broker at all, and fails at TLS with an error about an unknown authority.
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/,
`the builder was given nothing to verify the broker with:\n${credential}`);
// And it works: the mesh asks this builder to build something, and it does. Answering is the
// only proof that the delivered credential authenticates — a container that is up with a
// credential it cannot use looks identical from outside.
// Somewhere the builder can actually see. A builder that is a module runs in a container, so
// the machine's filesystem is not its own — a path like /root only works for a builder somebody
// started on the host, which is what the first build above used. In a real mesh a module is
// cloned from the forge over a URL; here it goes in the directory the module already mounts,
// which is the same fact wearing different clothes.
const repo = "/var/lib/mesh/builder/repositories/built";
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"built","version":"1",` +
`"resources":[{"id":"marker","type":"file","path":"/etc/built","content":"yes","mode":"0644"}]}' ` +
`> ${repo}/module.json`);
await must("anchor", `cd ${repo} && git init -q . && git add -A && ` +
`git -c user.email=lab -c user.name=lab commit -qm built`);
try {
await mesh(`build ${repo} --wait 300s`, 420_000);
} catch (why) {
// The builder's own account of itself. Without it the failure is "nothing consumed the
// queue", which names no cause and is the same sentence whether the credential was refused,
// the queue was never declared, or the process died three seconds in.
const said = (await on("anchor", `docker logs mesh-builder 2>&1 | tail -40`)).out;
throw new Error(`${(why as Error).message}\n\nwhat the builder said:\n${said}`);
}
// Naming the module, and not merely containing its name: `builds` says "nothing has been built
// yet" when there is nothing, and that sentence contains the word this was matching on.
const recorded = await mesh("builds built");
assert.doesNotMatch(recorded, /nothing has been built/,
`the build was accepted and no build was recorded against the module:\n${recorded}`);
assert.match(recorded, /built/, recorded);
});
test("rotating a credential moves both ends, and the old one stops working", { test("rotating a credential moves both ends, and the old one stops working", {
skip, timeout: 900_000, skip, timeout: 900_000,
@@ -992,7 +1072,7 @@ test("a route is a grant: a workload is reached by the name it asked for", {
// they are different questions: one says who may reach it, the other says by what name — and // they are different questions: one says who may reach it, the other says by what name — and
// the earlier test left this machine filtering, so a module that asked for a route and not for // the earlier test left this machine filtering, so a module that asked for a route and not for
// the port would be unreachable by the proxy it just asked for. // the port would be unreachable by the proxy it just asked for.
await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` + await must("anchor", `printf %s '{"module":"storefront","version":"1",` +
`"requires":["route"],"capabilities":["container-runtime"],` + `"requires":["route"],"capabilities":["container-runtime"],` +
`"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` + `"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
`"binds":{"route":"/etc/storefront/route.json"},` + `"binds":{"route":"/etc/storefront/route.json"},` +
@@ -1156,10 +1236,6 @@ test("a new commit reaches a machine that is already running the old one", {
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the // novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
// question "did my change go out?". This is that question, end to end — a commit, a build, a // question "did my change go out?". This is that question, end to end — a commit, a build, a
// catalogue, and a machine that ends up running what the source says. // catalogue, and a machine that ends up running what the source says.
// The hand-started builder does not outlive a broker restart, and the foundation's broker is
// recreated when the first push reconciles it: a builder is (re)started here, where a build is
// asked for. The mesh's own builder is a module with a restart policy and needs none of this.
await startBuilder();
const repo = "/var/lib/mesh/builder/repositories/delivered"; const repo = "/var/lib/mesh/builder/repositories/delivered";
const write = async (what: string) => const write = async (what: string) =>
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` + await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
@@ -1287,11 +1363,79 @@ test("the board names the machine that is not doing what it was told", {
}); });
// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries. // Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on test("the hub can be filtered without severing the mesh", {
// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a skip, timeout: 900_000,
// filter module derives the rules — so the credential test above assigns the catalogue's and }, async () => {
// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh // The machine that most needs a firewall was the one that could not have one. A hub is dialled
// was not severed. Retired 2026-09-22. // by every node at other sites; a machine that is not a hub dials out and needs nothing open.
// They are the same module, so a static `listens` cannot say it — and the machine it gets wrong
// is the one facing the public internet.
//
// The failure this guards against is not subtle and is very hard to recover from: a rule set
// that closes the hub's own port takes the private network down, and the mesh's way of fixing
// anything is to send a declaration over it.
// Its own directory. Another module on this machine already declares /etc/mesh, and the mesh
// refuses two modules declaring one path rather than letting the second quietly win — which it
// did here, correctly, the first time this ran.
const rules = "/etc/mesh-hub/filter.nft";
await must("anchor", `printf %s '{"module":"hubfilter","version":"1",` +
`"capabilities":["firewall"],` +
`"filtering":{"into":"${rules}"},` +
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
`{"id":"dir","type":"directory","path":"/etc/mesh-hub","mode":"0755"},` +
`{"id":"unit","type":"file","path":"/etc/systemd/system/hub-filter.service",` +
`"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for the hub\\n` +
`[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
`ExecStart=/usr/bin/nft -f ${rules}\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
`{"id":"filter","type":"service","unit":"hub-filter.service","state":"running",` +
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/hubfilter.json`);
await must("anchor", `docker cp /tmp/hubfilter.json mesh-controller:/hubfilter.json`);
await mesh("module add /hubfilter.json");
await mesh("assign anchor hubfilter");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 20_000));
// The hub's own way onto the private network is open, and derived — nothing in that manifest
// mentions a port.
const written = await must("anchor", `cat ${rules}`);
assert.match(written, /udp dport 51820 accept/,
`the hub's rule set closes the private network it is the way onto:\n${written}`);
// The module that provides the private network, not the requirement it answers: `networking`
// is the domain a module offers, and what caused a rule is the module itself.
assert.match(written, /# mesh-wireguard — the private network/,
`the rule does not name what caused it:\n${written}`);
// Loaded, and the mesh still works: a declaration reaches the other machine, which it cannot if
// the overlay is severed. This is the assertion that matters — a rule file that looks right and
// a mesh that has stopped are exactly what this is guarding against.
assert.match(await must("anchor", `nft list table inet mesh`), /dport 51820/);
await must("laptop", `rm -f /etc/mesh-still-works`);
await must("anchor", `printf %s '{"module":"stillworks","version":"1",` +
`"resources":[{"id":"marker","type":"file","path":"/etc/mesh-still-works",` +
`"content":"yes","mode":"0644"}]}' > /tmp/stillworks.json`);
await must("anchor", `docker cp /tmp/stillworks.json mesh-controller:/stillworks.json`);
await mesh("module add /stillworks.json");
await mesh("assign laptop stillworks");
await mesh("push laptop");
let arrived = false;
for (let i = 0; i < 20 && !arrived; i++) {
arrived = (await on("laptop", `test -f /etc/mesh-still-works`)).ok;
if (!arrived) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(arrived,
"the hub applied its own rule set and the mesh stopped reaching the other machine");
// And the other machine still reaches the hub over the private network, which is what the
// opened port is for.
assert.ok((await on("laptop", `ping -c 1 -W 5 anchor.internal`)).ok,
"the private network is down after the hub filtered itself");
await mesh("unassign anchor hubfilter");
await mesh("unassign laptop stillworks");
await mesh("push");
});
test("a container reaches another machine by the name the mesh gave it", { test("a container reaches another machine by the name the mesh gave it", {
skip, timeout: 900_000, skip, timeout: 900_000,
@@ -1336,23 +1480,6 @@ test("a container reaches another machine by the name the mesh gave it", {
// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being // Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
// told each one. // told each one.
/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so
* it is issued once per machine. The mesh writes the resolver's data as a fact the module
* declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */
const resolverIssued = new Set<string>();
async function resolverModules(machines: string[]): Promise<void> {
for (const name of ["dnsmasq", "resolved-split-dns"]) {
const manifest = catalogueModule(name, held);
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`);
}
for (const machine of machines) {
if (resolverIssued.has(machine)) continue;
await mesh(`module issue dnsmasq --node ${machine}`);
resolverIssued.add(machine);
}
}
test("every name under a machine resolves to that machine", { test("every name under a machine resolves to that machine", {
skip, timeout: 900_000, skip, timeout: 900_000,
}, async () => { }, async () => {
@@ -1363,11 +1490,9 @@ test("every name under a machine resolves to that machine", {
// //
// The mesh writes the data and runs no daemon: a resolver is third-party software, and the // The mesh writes the data and runs no daemon: a resolver is third-party software, and the
// mesh has no business choosing one. So what is checked here is the mesh's half — that the // mesh has no business choosing one. So what is checked here is the mesh's half — that the
// data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq // data is right, complete, and follows the machines.
// declares, so that module is what is assigned; what it runs is the next test's concern. await mesh("assign anchor mesh-resolver");
await resolverModules(["anchor", "laptop"]); await mesh("assign laptop mesh-resolver");
await mesh("assign anchor dnsmasq");
await mesh("assign laptop dnsmasq");
await mesh("push"); await mesh("push");
await new Promise((r) => setTimeout(r, 15_000)); await new Promise((r) => setTimeout(r, 15_000));
@@ -1393,10 +1518,10 @@ test("every name under a machine resolves to that machine", {
// because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name // because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
// fails at once and says which name it was. // fails at once and says which name it was.
// //
// Both: the resolver's data follows the private network, so the machine leaves the network as // Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the
// well as the resolver, and what is asserted is that the machine that stayed is answered for and // network, so unassigning the domain module alone leaves the machine on the network — pulled
// the one that left is not. // back by its own requirement. The mesh was right and this test was wrong the first time.
await mesh("unassign laptop dnsmasq"); await mesh("unassign laptop mesh-resolver");
await mesh("unassign laptop networking"); await mesh("unassign laptop networking");
await mesh("push anchor"); await mesh("push anchor");
await new Promise((r) => setTimeout(r, 15_000)); await new Promise((r) => setTimeout(r, 15_000));
@@ -1408,13 +1533,14 @@ test("every name under a machine resolves to that machine", {
`the machine that stayed lost its own name:\n${after}`); `the machine that stayed lost its own name:\n${after}`);
await mesh("assign laptop networking"); await mesh("assign laptop networking");
await mesh("unassign anchor dnsmasq"); await mesh("unassign anchor mesh-resolver");
await mesh("push"); await mesh("push");
await new Promise((r) => setTimeout(r, 15_000)); await new Promise((r) => setTimeout(r, 15_000));
}); });
test("a service is reached by a name under the machine it runs on", { test("a service is reached by a name under the machine it runs on", {
skip, timeout: 900_000, skip: skip || (!moduleExamples ? "set MESH_LAB_MODULES to mesh-controller's examples/modules" : false),
timeout: 900_000,
}, async () => { }, async () => {
// postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is // postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
// the node, so anything under a node's name must resolve to that node. What routes it once it // the node, so anything under a node's name must resolve to that node. What routes it once it
@@ -1426,7 +1552,12 @@ test("a service is reached by a name under the machine it runs on", {
// /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a // /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
// refusal rather than a fight over the file — and picking the wrong one here would have been // refusal rather than a fight over the file — and picking the wrong one here would have been
// testing that fight. // testing that fight.
await resolverModules(["anchor", "laptop"]); for (const name of ["dnsmasq", "resolved-split-dns"]) {
const manifest = readFileSync(`${moduleExamples}/${name}.json`, "utf8");
await must("anchor", `cat > /tmp/${name}.json <<'MANIFEST'\n${manifest}\nMANIFEST`);
await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`);
}
// Both machines, because a node resolves from its own copy — the same rule as everything else // Both machines, because a node resolves from its own copy — the same rule as everything else
// it holds. A mesh where one machine answers for all of them stops resolving when that machine // it holds. A mesh where one machine answers for all of them stops resolving when that machine
@@ -1540,7 +1671,7 @@ test("a third-party workload is adopted, with the credential it already had", {
const password = "the-password-it-already-had"; const password = "the-password-it-already-had";
await must("anchor", `printf %s ${quote(JSON.stringify({ await must("anchor", `printf %s ${quote(JSON.stringify({
module: "adopted-analytics", module: "umami",
version: "1", version: "1",
capabilities: ["container-runtime"], capabilities: ["container-runtime"],
"own-secrets": { "own-secrets": {
@@ -1576,13 +1707,13 @@ test("a third-party workload is adopted, with the credential it already had", {
// seals it and cannot read it again. Given whole, as the environment lines the containers read. // seals it and cannot read it again. Given whole, as the environment lines the containers read.
await must("anchor", await must("anchor",
`printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` + `printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
`docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics database --from -`); `docker exec -i mesh-controller /mesh-controller secret accept anchor umami database --from -`);
await must("anchor", await must("anchor",
`printf %s ${quote( `printf %s ${quote(
`DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` + `DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
`docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics app --from -`); `docker exec -i mesh-controller /mesh-controller secret accept anchor umami app --from -`);
await mesh("assign anchor adopted-analytics"); await mesh("assign anchor umami");
await mesh("push anchor", 300_000); await mesh("push anchor", 300_000);
// Both containers, and the network they share. // Both containers, and the network they share.
@@ -1653,12 +1784,305 @@ test("a third-party workload is adopted, with the credential it already had", {
// Running them needs their images stocked and two provisioners built, which is a separate and // Running them needs their images stocked and two provisioners built, which is a separate and
// larger job. This is the half that can be known now, and it is the half where a design fault // larger job. This is the half that can be known now, and it is the half where a design fault
// would live. // would live.
// Three tests lived here that read the mesh's example modules, which moved to the catalogue. test("the real modules resolve together, and compose a declaration a host accepts", {
// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood skip, timeout: 300_000,
// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the }, async (t) => {
// catalogue's modules together and its gate is the composed declaration accepted and every core // Everything the catalogue holds, except two whose names this mesh is already running under:
// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates // `registry` is the artifact store the suite stood up, and `umami` is the adopted workload —
// on gitea running but does not yet ask it to answer on its port with the credential it was given, // adding the catalogue's manifests would replace the records of modules that are live and
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy // assigned, and the adopted umami would suddenly require a database it never asked for.
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into const modules = ["postgres", "keycloak", "gitea", "minio", "mailu",
// mesh-grant-end-to-end, against the catalogue's redis. "redis", "grafana", "nextcloud", "searxng", "influxdb", "verdaccio"];
const planned: string[] = [];
for (const name of modules) {
const raw = readFileSync(
`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
// Pointed at this scenario's registry before being added, exactly as the forge is.
//
// **Not cosmetic.** Some of these name an image the mesh builds, whose digest does not exist
// until it is built — so the file legitimately carries a placeholder, and composing a
// declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is
// what this test used to do.
const pinned = pinnedInto(raw, held);
// What this scenario does not serve cannot be redirected, and a module still naming a
// placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped
// and said, rather than silently dropped: a planning test quietly covering four modules
// instead of five is the false coverage this suite exists to prevent.
const left = stillUnpinned(pinned);
if (left.length > 0) {
console.log(`skipping ${name}: this scenario serves no ${left.join(", ")}`);
continue;
}
planned.push(name);
await must("anchor", `printf %s ${quote(pinned)} > /${name}.json`);
await must("anchor", `docker cp /${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`);
}
// **Put the machine back whatever happens.** Tests here share one mesh, so what this one
// assigns is what the next one inherits. Written at the end of the body once, it was skipped
// the first time this test failed — and the next test's push was refused by a module this one
// had left behind, which reads as a fault in the test that was actually working.
t.after(async () => {
for (const name of planned) await mesh(`unassign anchor ${name}`).catch(() => {});
});
// Assigned one at a time, because assignment resolves the whole set and says so immediately.
// A refusal here is the graph rejecting something, which is the point of asking.
for (const name of planned) {
await mesh(`assign anchor ${name}`);
}
const plan = await mesh("plan anchor --json", 120_000);
const declaration = JSON.parse(plan.slice(plan.indexOf("{")));
const byId = new Map<string, any>(
(declaration.resources as any[]).map((r) => [r.id, r]));
const ids = [...byId.keys()];
// Every module's own network, which only exists because more than one container needs to reach
// another by name.
for (const id of ["postgres.net", "keycloak.net", "minio.net", "mailu.net"]) {
assert.ok(byId.has(id), `${id} is missing; ${ids.length} resources: ${ids.join(", ")}`);
assert.equal(byId.get(id).type, "network");
}
// The cross-module edge: keycloak asked for a database and was told where it is and given a
// credential. Neither file is anything keycloak's manifest could have written.
const bound = [...byId.values()].find((r) =>
r.type === "file" && r.path === "/var/lib/keycloak/database.json");
assert.ok(bound, `keycloak was never told where its database is: ${ids.join(", ")}`);
assert.match(JSON.stringify(bound), /postgres/,
"keycloak's binding does not name what answered its requirement");
// The password, alone in a file and sealed. It is a password and nothing else, so nothing reads
// it as configuration — novox/hq 04-ISSUES/023 and the playbook both turn on that distinction.
const credential = [...byId.values()].find((r) =>
r.type === "file" && r.path === "/var/lib/keycloak/database.secret");
assert.ok(credential, `keycloak was given no credential for its database: ${ids.join(", ")}`);
assert.ok(credential.sealed, "keycloak's credential is not sealed, so the mesh can read it");
assert.ok(!credential.content, "a credential arrived as content rather than sealed");
// And the connection itself, which keycloak could not have written: the address and port come
// from what the provider serves, and the user name from what the mesh decided both ends would
// call this consumer (novox/hq 04-ISSUES/023).
const connection = [...byId.values()].find((r) =>
r.type === "file" && r.path === "/var/lib/keycloak/database.env");
assert.ok(connection, "keycloak was given no database configuration");
assert.match(connection.content, /KC_DB_USERNAME=mesh_[a-z0-9_]+_keycloak/,
`keycloak was not told what name to present:\n${connection.content}`);
assert.doesNotMatch(connection.content, /\$\{bound:/,
`a placeholder reached the machine as a value:\n${connection.content}`);
// The password is the one hole left open, and the sealed value travels beside it. The mesh
// discarded the plaintext, so the host is the only thing that can close it.
assert.match(connection.content, /KC_DB_PASSWORD=\$\{secret:postgres-database\}/,
`the password was not left for the host to fill:\n${connection.content}`);
assert.ok(connection.secrets?.["postgres-database"],
"the sealed credential did not travel with the file that needs it");
assert.doesNotMatch(JSON.stringify(connection.content), /postgres-database":"[A-Za-z0-9+/]{24,}/,
"the credential was written into the configuration in the clear");
// And the provider was told who asked, which is what its provisioner reconciles against.
const grants = [...byId.values()].find((r) =>
r.type === "file" && String(r.path).startsWith("/var/lib/postgres/grants"));
assert.ok(grants, "postgres was never told which modules were granted a database");
assert.match(JSON.stringify(grants), /keycloak|gitea/,
"the grants file names neither module that asked for a database");
// Secrets reach containers as files, never as environment in the declaration.
const containers = [...byId.values()].filter((r) => r.type === "container");
assert.ok(containers.length >= 12,
`only ${containers.length} containers; mailu alone is nine`);
for (const c of containers) {
for (const [key, value] of Object.entries(c.env ?? {})) {
// **An absolute path is a reference to a secret, not a secret**, and naming one is the
// whole design: the mesh delivers a credential as a file and a module says where.
//
// Excluded because `/` is in the base64 alphabet, so any path of 24 characters or more
// matched — `MESH_BROKER_FILE=/var/lib/mesh/builder/broker` was reported as a credential
// the broker would see. A check that fires on the right shape for the wrong reason is
// worse than none: it is the one that gets suppressed, and then it is not there when it
// is right.
if (String(value).startsWith("/")) continue;
assert.doesNotMatch(String(value), /^[A-Za-z0-9+/]{24,}={0,2}$/,
`${c.name} carries something secret-shaped in env.${key}, which the broker would see`);
}
}
});
// The first of the real module descriptions to actually run.
//
// **Everything before this stopped at composing a declaration.** That proves the control plane and
// the host agree, and proves nothing about whether the thing described works — which is how five
// modules sat pinned to images that did not exist, parsing and resolving perfectly
// (novox/hq 04-ISSUES/025).
//
// The forge is the one worth running first. It needs a database from another module, a password it
// did not choose, and a connection string it could not have written itself: the address and port
// come from what the database serves, and the user name from what the mesh decided both ends would
// call it (04-ISSUES/022 and 023). If any of that is wrong it cannot start, and nothing else in
// this file would notice.
test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 }, async () => {
for (const name of ["postgres", "gitea"]) {
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
// An image the mesh builds has no digest until it is built, and one it does not build belongs
// to whichever registry served it. Only the first is rewritten; the second is pulled.
const pinned = pinnedInto(raw, held);
assert.deepEqual(stillUnpinned(pinned), [],
`${name} still names an image nothing serves, so it could not start`);
await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`);
await must("anchor", `docker cp /run-${name}.json mesh-controller:/run-${name}.json`);
await mesh(`module add /run-${name}.json`);
await mesh(`assign anchor ${name}`);
}
await mesh("push anchor", 300_000);
// **What the machine says it did, before asking what it produced.** This test pushed and then
// waited for a database role, so when the containers were never created at all it reported "no
// login was created" — true, and silent about the reason. A push that was accepted and an apply
// that worked are different facts, and the second is the one this depends on.
//
// And waited for, because `push` sends without waiting. Reading `status` the instant it returns
// describes the apply *before* this one, which is how this test came to report a missing
// container while insisting the machine was fine.
await settled("anchor");
const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out;
// Named with what the mesh meant to send, not only with what the machine has. A container that
// is absent because the mesh never asked for it and one that is absent because the machine could
// not make it are the same sentence here and different faults entirely, and the plan is the only
// thing that tells them apart.
assert.match(running, /\bpostgres\b/,
`the database module was pushed and no container for it exists:\n${running}\n\n` +
`what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` +
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
// The database first: until the provisioner has made the login, the forge has nothing to
// connect to and its own start would prove only that it retries.
const psql = async (q: string) =>
(await on("anchor",
`docker exec postgres psql -U postgres -qAt -c ${quote(q)}`, 60_000)).out.trim();
// Both halves in one poll. The provisioner makes the role and then the database, and a test
// that waited for the first and checked the second once was racing the gap between two
// statements — it lost, once, eighteen seconds into a run.
let made = "";
for (let i = 0; i < 40 && made !== "t"; i++) {
made = await psql("select true from pg_roles where rolname = 'mesh_anchor_gitea'" +
" and exists (select from pg_database where datname = 'gitea')");
if (made !== "t") await new Promise((r) => setTimeout(r, 3000));
}
assert.equal(made, "t",
`no login was created for the forge:\n${(await on("anchor", "docker logs mesh-provision-postgres 2>&1 | tail -20")).out}`);
// And the forge itself, answering. Not that its container exists — that it serves.
//
// On the port the mesh assigned, not the one the module declared (novox/hq ADR 0038): the
// module says 3000 and the machine publishes wherever the mesh put it. Read from the plan,
// because the plan is the same composition a push sends.
const planned = await mesh("plan anchor --json", 120_000);
const mapping = (JSON.parse(planned.slice(planned.indexOf("{"))).resources as any[])
.find((r) => r.id === "gitea.server")?.ports
?.map(String).find((p: string) => p.endsWith(":3000"));
assert.ok(mapping, "the plan does not say where the machine publishes the forge");
const at = mapping.split(":")[0];
let answered = false;
let said = { out: "", ok: false };
for (let i = 0; i < 60 && !answered; i++) {
said = await on("anchor", `curl -sf -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${at}/`, 30_000);
answered = said.out.trim().startsWith("2") || said.out.trim() === "303";
if (!answered) await new Promise((r) => setTimeout(r, 5000));
}
assert.ok(answered,
`the forge never answered (last: ${said.out.trim()}):\n` +
`${(await on("anchor", "docker logs gitea 2>&1 | tail -25")).out}`);
// **The credential actually worked.** A forge that started and could not reach its database
// would still answer on its port, so the log is where the difference lives.
const log = (await on("anchor", "docker logs gitea 2>&1 | tail -60")).out;
assert.doesNotMatch(log, /password authentication failed|connection refused|does not exist/i,
`the forge started and could not use the database it was given:\n${log}`);
await mesh("unassign anchor gitea");
await mesh("unassign anchor postgres");
await mesh("push anchor", 300_000);
});
test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600_000 }, async (t) => {
// The third provision after a database and a bucket, and the first whose tenancy is enforced
// by the store's own ACL rather than by separate namespaces: every consumer shares one
// keyspace, so the grant is a pattern — and the test is that the pattern means what the
// manifest said, in both directions.
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8");
const pinned = pinnedInto(raw, held);
assert.deepEqual(stillUnpinned(pinned), [],
"redis still names an image nothing serves, so it could not start");
await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`);
await must("anchor", `docker cp /run-redis.json mesh-controller:/run-redis.json`);
await mesh("module add /run-redis.json");
// A consumer with no container: what is under test is the credential's reach, and files on the
// machine are enough to prove it — the same reduction the first credential test makes.
await must("anchor", `printf %s '{"module":"cachetest","version":"1",` +
`"requires":["redis-cache"],` +
`"contributes":{"redis-cache":{"prefix":"cachetest"}},` +
`"binds":{"redis-cache":"/var/lib/cachetest/cache.json"},` +
`"secrets":{"redis-cache":"/var/lib/cachetest/cache.secret"},` +
`"resources":[{"id":"state","type":"directory","path":"/var/lib/cachetest","mode":"0700"}]}' ` +
`> /cachetest.json`);
await must("anchor", `docker cp /cachetest.json mesh-controller:/cachetest.json`);
await mesh("module add /cachetest.json");
await mesh("assign anchor redis");
await mesh("assign anchor cachetest");
await mesh("push anchor", 300_000);
await settled("anchor");
t.after(async () => {
for (const name of ["cachetest", "redis"]) {
await mesh(`unassign anchor ${name}`).catch(() => {});
}
await mesh("push anchor", 300_000).catch(() => {});
});
// What the mesh told each end. The consumer's user name comes from its binding; the user's
// password from the sealed file beside it — both written by the host, neither invented here.
const bound = JSON.parse(await must("anchor", `cat /var/lib/cachetest/cache.json`));
const user = bound.as;
assert.ok(user?.startsWith("mesh_"), `the binding does not carry a usable user: ${user}`);
const secret = (await must("anchor", `cat /var/lib/cachetest/cache.secret`)).trim();
// The provisioner has to have run before anything can authenticate. Waited for via the store
// itself: the user list, asked with the server's own password, which the conf file the host
// wrote holds on the machine.
const admin = (await must("anchor",
`awk '/^requirepass/ {print $2}' /var/lib/redis-module/redis.conf`)).trim();
let granted = false;
for (let i = 0; i < 40 && !granted; i++) {
const users = (await on("anchor",
`docker exec redis redis-cli --no-auth-warning -a ${quote(admin)} ACL USERS`)).out;
granted = users.includes(user);
if (!granted) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(granted, `no user was created for the consumer:
` +
`containers:\n${(await on("anchor", "docker ps -a --format '{{.Names}} {{.Status}}' | head -20")).out}\n` +
`the store:\n${(await on("anchor", "docker logs redis 2>&1 | tail -15")).out}\n` +
`the provisioner:\n${(await on("anchor", "docker logs mesh-provision-redis 2>&1 | tail -15")).out}`);
const asConsumer = (command: string) =>
on("anchor", `docker exec redis redis-cli --no-auth-warning ` +
`--user ${quote(user)} --pass ${quote(secret)} ${command}`);
// Its own keys: usable.
assert.match((await asConsumer("SET cachetest:proof yes")).out, /OK/,
"the consumer cannot write under the prefix it was granted");
assert.match((await asConsumer("GET cachetest:proof")).out, /yes/,
"the consumer cannot read back what it wrote");
// Anyone else's: refused by the store itself, which is the entire point of the grant.
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
"the consumer wrote outside its prefix — the grant means more than the manifest said");
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
"the consumer can flush the store, which no tenant may");
});
@@ -0,0 +1,252 @@
/**
* The whole grant for an S3 bucket, mesh-driven — novox/hq ADR 0052/0053, the minio case.
*
* The postgres bed proves the provider/consumer contract for a database. This proves it for object
* storage, on a provider whose code drives the `mc` CLI (so the runtime image carries it): minio is
* assigned, a consumer that requires s3-bucket is assigned, and the mesh mints one secret key, sealing
* a copy to each end. minio's provisioner — reading only the mesh's contributions — creates a bucket
* and a service account under the access key the mesh derived, with the secret it minted. The proof is
* the consumer reaching its bucket with the access key and secret the mesh delivered it. Nothing is
* placed by the test.
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh minio builds mesh-runtime-minio:development (with mc), which
* scenarios/minio-node.yml stocks. minio/minio:latest must be in the local daemon.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
// 04-ISSUES/010 is fixed by ADR 0054: an S3 access key is capped at 20, and the mesh derives
// `mesh_<node>_<module>`, so `bucketuser` on `anchor` (22) would overflow — but a consumer declares a
// short `slug` and its identity fits. This bed's consumer does exactly that.
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "minio-node";
const MACHINE = "anchor";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
/** Same rule minio's client uses to name a bucket for a consumer — recomputed so the test knows it. */
function bucketFor(as: string): string {
const name = as.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63);
return name.length >= 3 ? name : `mesh-${name}`;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh grants a consumer an S3 bucket, and the credential it delivers reaches it", {
skip, timeout: 900_000,
}, async () => {
// The PROVIDER: minio in its committed shape — server and a broker-bound runtime (carrying mc) on
// the private minio network, the runtime running the provisioner. No published port on this
// single-node bed; the consumer reaches minio over the private network by name.
const minioManifest = JSON.stringify({
module: "minio",
version: "1",
provides: [{ name: "s3-bucket", scope: "mesh" }],
serves: { "s3-bucket": { scheme: "http", region: "us-east-1" } },
emits: ["module.minio.bucket.created", "module.minio.bucket.removed"],
receives: { "s3-bucket": "/var/lib/minio/grants/mesh.json" },
grants: { "s3-bucket": "/var/lib/minio/grants" },
"own-secrets": { root: "/var/lib/minio/root.secret", broker: "/var/lib/mesh/minio/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/minio", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/minio", mode: "0700" },
{ id: "grants", type: "directory", path: "/var/lib/minio/grants", mode: "0700" },
{ id: "root-env", type: "file", path: "/var/lib/minio/root.env", mode: "0600", content: "MINIO_ROOT_USER=meshroot\nMINIO_ROOT_PASSWORD=${secret:root}\n" },
{ id: "data", type: "directory", path: "/services/minio/data/data1-1", mode: "0700" },
{ id: "net", type: "network", name: "minio" },
{
id: "server", type: "container", name: "minio", image: pinned("minio/minio"), network: "minio",
args: ["server", "/data", "--console-address", ":9001"],
"env-file": ["/var/lib/minio/root.env"],
volumes: ["/services/minio/data/data1-1:/data"],
},
{
id: "runtime", type: "container", name: "mesh-minio", image: pinned("mesh-runtime-minio"),
network: "minio",
volumes: [
"/var/lib/mesh/minio/broker:/run/secrets/broker:ro",
"/var/lib/minio/grants:/var/lib/minio/grants:ro",
"/var/lib/minio/root.secret:/run/secrets/root:ro",
],
env: {
MESH_MINIO_ENDPOINT: "http://minio:9000",
MESH_MINIO_ROOT_USER: "meshroot",
MESH_MINIO_ROOT_PASSWORD_FILE: "/run/secrets/root",
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/minio/grants/mesh.json",
},
},
],
});
const consumerManifest = JSON.stringify({
module: "bucketuser",
version: "1",
// A short slug, so the derived identity `mesh_anchor_bkt` fits an S3 access key's 20 chars where
// `mesh_anchor_bucketuser` (22) would not (novox/hq ADR 0054, 04-ISSUES/010).
slug: "bkt",
requires: ["s3-bucket"],
contributes: { "s3-bucket": { name: "bucketuser" } },
binds: { "s3-bucket": "/var/lib/bucketuser/s3.json" },
secrets: { "s3-bucket": "/var/lib/bucketuser/s3.secret" },
resources: [{ id: "state", type: "directory", path: "/var/lib/bucketuser", mode: "0700" }],
});
await must(`printf %s ${quote(minioManifest)} > /tmp/minio.json && docker cp /tmp/minio.json mesh-controller:/minio.json`);
await mesh("module add /minio.json");
await mesh(`module issue minio --node ${MACHINE}`);
await mesh(`assign ${MACHINE} minio`);
await must(`printf %s ${quote(consumerManifest)} > /tmp/bucketuser.json && docker cp /tmp/bucketuser.json mesh-controller:/bucketuser.json`);
await mesh("module add /bucketuser.json");
await mesh(`assign ${MACHINE} bucketuser`);
await mesh(`push ${MACHINE}`);
await settled();
// The mesh delivered the consumer its bound file and its unsealed secret.
let boundRaw = "";
const untilBound = Date.now() + 60_000;
while (Date.now() < untilBound) {
const got = await on(`cat /var/lib/bucketuser/s3.json 2>/dev/null`);
if (got.ok && /"as"/.test(got.out)) { boundRaw = got.out; break; }
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(boundRaw, /"as"/, `the consumer was never told about its bucket:\n${boundRaw}`);
const bound = JSON.parse(boundRaw) as { as: string; provision: string };
assert.equal(bound.provision, "s3-bucket");
const accessKey = bound.as;
const secretKey = (await must(`cat /var/lib/bucketuser/s3.secret`)).trim();
assert.ok(accessKey && secretKey, `the consumer's access key or secret was empty (as=${accessKey})`);
const bucket = bucketFor(accessKey);
// THE PROOF: reach the bucket as the consumer, with the access key and secret the mesh delivered
// it. mc listing the consumer's own bucket means the service account, the bucket, and the secret all
// line up across the two ends. A provisioner that set a different secret answers "Access Denied".
const probe =
`mc alias set probe http://minio:9000 ${quote(accessKey)} ${quote(secretKey)} >/dev/null 2>&1 && ` +
`mc ls probe/${quote(bucket)}/`;
let out = { out: "", ok: false };
const untilReach = Date.now() + 90_000;
while (Date.now() < untilReach) {
out = await on(`docker exec mesh-minio sh -c ${quote(probe)} 2>&1`);
if (out.ok) break;
if (/denied/i.test(out.out)) break; // fast-fail: the credential is wrong
await new Promise((r) => setTimeout(r, 3000));
}
assert.doesNotMatch(out.out, /denied/i,
`the consumer could not reach its bucket with the mesh's secret — the two ends do not agree:\n${out.out}`);
assert.ok(out.ok,
`the consumer could not list its granted bucket ${bucket} as ${accessKey}:\n${out.out}\n---\n${(await on(`docker logs mesh-minio 2>&1 | tail -30`)).out}`);
});
+28 -6
View File
@@ -24,7 +24,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -37,7 +37,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "openai-bed"; const SCENARIO = "openai-bed";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -156,6 +156,7 @@ after(async () => {
test("a static-key model-access licence delivers the operator's API key to the consumer, unchanged", { test("a static-key model-access licence delivers the operator's API key to the consumer, unchanged", {
skip, timeout: 1_500_000, skip, timeout: 1_500_000,
}, async () => { }, async () => {
const consumerImage = pinned("mesh-runtime-openai-consumer");
// --- the licence, a record with vendor openai (static-key) ------------------------------------- // --- the licence, a record with vendor openai (static-key) -------------------------------------
// No manager: a static-key licence has none (mesh-controller refuses `licence manager` on it). The // No manager: a static-key licence has none (mesh-controller refuses `licence manager` on it). The
@@ -171,15 +172,36 @@ test("a static-key model-access licence delivers the operator's API key to the c
await mesh(`licence key personal --file /openai-key`); await mesh(`licence key personal --file /openai-key`);
// --- deploy the consumer ----------------------------------------------------------------------- // --- deploy the consumer -----------------------------------------------------------------------
// The catalogue's own manifest (novox/hq 04-ISSUES/073): a model-access holder whose delivered key // Inline manifest mirroring the committed module.json: a model-access holder whose delivered key
// arrives at its secret path. The scheduled apply container installs as present state (ADR 0053) and // arrives at its secret path. The scheduled apply container installs as present state (ADR 0053) and
// its image is pulled at apply (schedule-pull); the test drives apply directly for a deterministic // its image is pulled at apply (schedule-pull); the test drives apply directly for a deterministic
// flow rather than waiting on cron. // flow rather than waiting on cron.
const consumerManifest = catalogueModule("openai-consumer", held); const consumerManifest = JSON.stringify({
module: "openai-consumer",
version: "1",
requires: ["model-access"],
binds: { "model-access": "/var/lib/openai-consumer/model.json" },
secrets: { "model-access": "/var/lib/openai-consumer/api-key" },
resources: [
{ id: "state", type: "directory", path: "/var/lib/openai-consumer", mode: "0700" },
{ id: "config", type: "directory", path: "/var/lib/openai-consumer/config", mode: "0700" },
{
id: "apply", type: "container", name: "mesh-openai-consumer-apply",
image: consumerImage, network: "host", schedule: "*/5 * * * *",
args: ["run", "/app/modules/openai-consumer/dist/apply/index.js"],
volumes: ["/var/lib/openai-consumer:/run/state"],
env: {
MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/api-key",
MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
MESH_OPENAI_ENV_FILE: "/run/state/config/openai.env",
MESH_OPENAI_CREDENTIALS_FILE: "/run/state/config/auth.json",
},
},
],
});
await must(`printf %s ${quote(consumerManifest)} > /tmp/openai-consumer.json && docker cp /tmp/openai-consumer.json mesh-controller:/openai-consumer.json`); await must(`printf %s ${quote(consumerManifest)} > /tmp/openai-consumer.json && docker cp /tmp/openai-consumer.json mesh-controller:/openai-consumer.json`);
await mesh(`module add /openai-consumer.json`); await mesh(`module add /openai-consumer.json`);
// No `module issue`: the consumer speaks on no bus, and issuing a module with no broker secret await mesh(`module issue openai-consumer --node ${MACHINE}`);
// to deliver into is refused (novox/hq issue 078).
await mesh(`assign ${MACHINE} openai-consumer`); await mesh(`assign ${MACHINE} openai-consumer`);
await mesh(`push ${MACHINE}`); await mesh(`push ${MACHINE}`);
await settled(); await settled();
@@ -0,0 +1,241 @@
/**
* The whole grant for a database, mesh-driven — novox/hq ADR 0052/0053, the postgres case.
*
* The redis bed proves the provider/consumer contract for a cache. This proves it for a database, on
* a provider whose code shells out to `psql` (so the runtime image carries it): postgres is assigned,
* a consumer that requires postgres-database is assigned, and the mesh mints one password, seals a
* copy to each end, and writes each its file. postgres's provisioner — reading only the mesh's
* contributions — creates a role and a database under the login the mesh derived, with the password
* the mesh minted. The proof is the consumer connecting to its database with the credential the mesh
* delivered it. Nothing is placed by the test.
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh postgres builds mesh-runtime-postgres:development (with psql),
* which scenarios/postgres-node.yml stocks.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "postgres-node";
const MACHINE = "anchor";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("the mesh grants a consumer a postgres database, and the credential it delivers connects", {
skip, timeout: 900_000,
}, async () => {
// The PROVIDER: postgres in its committed shape — server and a broker-bound runtime (carrying psql)
// on the private postgres network, the runtime running the provisioner.
const postgresManifest = JSON.stringify({
module: "postgres",
version: "1",
provides: [{ name: "postgres-database", scope: "mesh" }],
serves: { "postgres-database": {} },
emits: ["module.postgres.database.provisioned", "module.postgres.database.deprovisioned"],
consumes: ["module.postgres.database.provisioned", "module.postgres.database.deprovisioned"],
receives: { "postgres-database": "/var/lib/postgres/grants/mesh.json" },
grants: { "postgres-database": "/var/lib/postgres/grants" },
"own-secrets": { superuser: "/var/lib/postgres/superuser.secret", broker: "/var/lib/mesh/postgres/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/postgres", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/postgres", mode: "0700" },
{ id: "grants", type: "directory", path: "/var/lib/postgres/grants", mode: "0700" },
{ id: "superuser-env", type: "file", path: "/var/lib/postgres/superuser.env", mode: "0600", content: "POSTGRES_PASSWORD=${secret:superuser}\n" },
{ id: "data", type: "directory", path: "/services/postgres/db-data", mode: "0700" },
{ id: "net", type: "network", name: "postgres" },
{
// No published port here: the foundation's own store already holds host :5432 on this
// single-node bed, and the consumer reaches postgres over the private network by name. The
// committed manifest publishes it for cross-node consumers, which is a different node.
id: "server", type: "container", name: "postgres", image: pinned("postgres"), network: "postgres",
env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres" },
"env-file": ["/var/lib/postgres/superuser.env"],
volumes: ["/services/postgres/db-data:/var/lib/postgresql/data"],
},
{
id: "runtime", type: "container", name: "mesh-postgres", image: pinned("mesh-runtime-postgres"),
network: "postgres",
volumes: [
"/var/lib/mesh/postgres/broker:/run/secrets/broker:ro",
"/var/lib/postgres/grants:/var/lib/postgres/grants:ro",
"/var/lib/postgres/superuser.secret:/run/secrets/superuser:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/postgres/grants/mesh.json",
MESH_PROVISION_POSTGRES: "postgres://postgres@postgres:5432/postgres?sslmode=disable",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/superuser",
},
},
],
});
// The CONSUMER: a module that requires postgres-database and contributes a name so it asks.
const consumerManifest = JSON.stringify({
module: "dbuser",
version: "1",
requires: ["postgres-database"],
contributes: { "postgres-database": { name: "dbuser" } },
binds: { "postgres-database": "/var/lib/dbuser/db.json" },
secrets: { "postgres-database": "/var/lib/dbuser/db.secret" },
resources: [{ id: "state", type: "directory", path: "/var/lib/dbuser", mode: "0700" }],
});
await must(`printf %s ${quote(postgresManifest)} > /tmp/postgres.json && docker cp /tmp/postgres.json mesh-controller:/postgres.json`);
await mesh("module add /postgres.json");
await mesh(`module issue postgres --node ${MACHINE}`);
await mesh(`assign ${MACHINE} postgres`);
await must(`printf %s ${quote(consumerManifest)} > /tmp/dbuser.json && docker cp /tmp/dbuser.json mesh-controller:/dbuser.json`);
await mesh("module add /dbuser.json");
await mesh(`assign ${MACHINE} dbuser`);
await mesh(`push ${MACHINE}`);
await settled();
// The mesh delivered the consumer its bound file and its unsealed password.
let boundRaw = "";
const untilBound = Date.now() + 60_000;
while (Date.now() < untilBound) {
const got = await on(`cat /var/lib/dbuser/db.json 2>/dev/null`);
if (got.ok && /"as"/.test(got.out)) { boundRaw = got.out; break; }
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(boundRaw, /"as"/, `the consumer was never told about its database:\n${boundRaw}`);
const bound = JSON.parse(boundRaw) as { as: string; provision: string };
assert.equal(bound.provision, "postgres-database");
const as = bound.as;
const password = (await must(`cat /var/lib/dbuser/db.secret`)).trim();
assert.ok(as && password, `the consumer's login or password was empty (as=${as})`);
// THE PROOF: connect to postgres as the consumer, with the login and password the mesh delivered
// it, to the database postgres's provisioner created — a real password-checked TCP connection (the
// runtime carries psql). A `1` back means the role, the database, and the password all line up
// across the two ends. A provisioner that set a different password answers "authentication failed".
const conn = `postgresql://${as}:${encodeURIComponent(password)}@postgres:5432/${as}?sslmode=disable`;
let out = { out: "", ok: false };
const untilConn = Date.now() + 90_000;
while (Date.now() < untilConn) {
out = await on(`docker exec mesh-postgres psql ${quote(conn)} -tAc 'select 1' 2>&1`);
if (out.ok && /^1$/m.test(out.out)) break;
if (/authentication failed/i.test(out.out)) break; // fast-fail: the credential is wrong
await new Promise((r) => setTimeout(r, 3000));
}
assert.doesNotMatch(out.out, /authentication failed/i,
`the consumer could not authenticate with the mesh's password — the two ends do not agree:\n${out.out}`);
assert.match(out.out, /^1$/m,
`the consumer could not connect to its granted database as ${as}:\n${out.out}\n---\n${(await on(`docker logs mesh-postgres 2>&1 | tail -30`)).out}`);
});
@@ -9,7 +9,7 @@
* is on the broker — none of which happens if it could not reach the broker from the bridge. * is on the broker — none of which happens if it could not reach the broker from the bridge.
* *
* This mirrors the redis committed manifest exactly (server on `redis` with a published port, runtime * This mirrors the redis committed manifest exactly (server on `redis` with a published port, runtime
* on `redis`), where the earlier host-networked bed (since retired) took the shortcut. If this is green, the * on `redis`), where provider-uses-mesh-credential used host networking. If this is green, the
* private-network shape is the one to roll out to every provider. * private-network shape is the one to roll out to every provider.
*/ */
@@ -20,7 +20,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -33,7 +33,7 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "redis-node"; const SCENARIO = "redis-node";
const MACHINE = "anchor"; const MACHINE = "anchor";
@@ -136,15 +136,54 @@ after(async () => {
test("redis's runtime, on the backend's private network, binds the broker and provisions with the mesh's credential", { test("redis's runtime, on the backend's private network, binds the broker and provisions with the mesh's credential", {
skip, timeout: 900_000, skip, timeout: 900_000,
}, async () => { }, async () => {
// The catalogue's redis (novox/hq 04-ISSUES/074): a private `redis` network, the server on it // Exactly the committed redis shape: a private `redis` network, the server on it with a published
// with a published port, and the runtime on it too — reaching redis by name and the broker by // port, and the runtime on it too — reaching redis by name and the broker by NAT.
// NAT. Its own password is a `secret` the vault provides, so the vault is installed beside it. const manifest = JSON.stringify({
const vaultManifest = catalogueModule("mesh-vault", held); module: "redis",
await must(`printf %s ${quote(vaultManifest)} > /tmp/mesh-vault.json && docker cp /tmp/mesh-vault.json mesh-controller:/mesh-vault.json`); version: "1",
await mesh("module add /mesh-vault.json"); provides: [{ name: "redis-cache", scope: "mesh" }],
await mesh(`module issue mesh-vault --node ${MACHINE}`); serves: { "redis-cache": {} },
await mesh(`assign ${MACHINE} mesh-vault`); emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
const manifest = catalogueModule("redis", held); consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
grants: { "redis-cache": "/var/lib/redis-module/grants" },
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
{
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
},
{ id: "net", type: "network", name: "redis" },
{
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "redis",
ports: ["6379"],
volumes: [
"/services/redis/data:/data",
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
],
args: ["/etc/redis/redis.conf"],
},
{
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
network: "redis",
volumes: [
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
"/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro",
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
MESH_PROVISION_REDIS: "redis:6379",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`); await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
await mesh("module add /redis.json"); await mesh("module add /redis.json");
await mesh(`module issue redis --node ${MACHINE}`); await mesh(`module issue redis --node ${MACHINE}`);
@@ -0,0 +1,236 @@
/**
* A provider creates the resource with the credential the mesh minted — novox/hq ADR 0048.
*
* The old provisioner generated its own password, sealed it with a key nothing delivered, and
* handed it back. This proves the corrected contract: redis's provisioner reads the mesh's
* contributions file and, for each consumer, the password the mesh minted and the host unsealed, and
* creates the ACL user under the login the mesh derived, with that exact password. No $MESH_SEAL_KEY
* is set anywhere. The proof is authentication: a client logging in as that consumer with the mesh's
* password gets PONG — where a provisioner that invented its own password would answer WRONGPASS.
*
* A hand-written contributions file and secret stand in for the control plane here (a full grant
* from a second module is a heavier bed); their SHAPE is exactly what mesh-controller writes — a
* `receives` doc with `as`/`secret`, and the secret file the host leaves after unsealing.
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
* scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development, which
* scenarios/redis-node.yml stocks.
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "redis-node";
const MACHINE = "anchor";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, MACHINE, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(command, timeoutMs);
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function settled(withinMs = 480_000): Promise<void> {
const until = Date.now() + withinMs;
let last = "";
while (Date.now() < until) {
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === MACHINE);
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === MACHINE);
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5000));
}
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must(`docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh(`node add ${MACHINE}`);
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("redis creates a consumer's login with the password the mesh minted, sealing nothing", {
skip, timeout: 900_000,
}, async () => {
// redis as a provider: the server, and a broker-bound runtime that serves its tools AND runs its
// provisioner. The provisioner is pointed at the contributions file the mesh would write
// (MESH_RECEIVES). There is NO MESH_SEAL_KEY — the whole point of ADR 0048 is that a provider
// needs none.
const manifest = JSON.stringify({
module: "redis",
version: "1",
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
{
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
},
{
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
volumes: [
"/services/redis/data:/data",
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
],
args: ["/etc/redis/redis.conf"],
},
{
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
network: "host",
volumes: [
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/redis-module/grants/redis-cache.json",
MESH_PROVISION_REDIS: "127.0.0.1:6379",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
},
},
],
});
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
await mesh("module add /redis.json");
await mesh(`module issue redis --node ${MACHINE}`);
await mesh(`assign ${MACHINE} redis`);
await mesh(`push ${MACHINE}`);
await settled();
const running = await must(`docker ps --format '{{.Names}}'`);
assert.match(running, /mesh-redis/, `redis's runtime is not running:\n${(await on(`docker logs mesh-redis 2>&1 | tail -20`)).out}`);
// What the mesh delivers to the provider: a contributions file naming the consumer's login and
// where its password is, and the password itself as the file the host leaves after unsealing.
const password = "mesh-minted-9f3c2a";
await must(`printf %s ${quote(password)} > /var/lib/redis-module/grants/app.secret`);
const contributions = JSON.stringify({
contributions: 1,
requirement: "redis-cache",
generated: "by the mesh — do not edit",
given: [
{ from: "app", node: "app-node", at: "192.0.2.20:6379", as: "app-one", secret: "/var/lib/redis-module/grants/app.secret", values: {} },
],
});
await must(`printf %s ${quote(contributions)} > /var/lib/redis-module/grants/redis-cache.json`);
// Within a reconcile tick the provisioner creates the ACL user. It exists on the server.
let acl = "";
const until = Date.now() + 60_000;
while (Date.now() < until) {
acl = (await on(`docker exec redis redis-cli -a ${quote(await must(`cat /var/lib/redis-module/default.secret`))} --no-auth-warning ACL LIST 2>/dev/null`)).out;
if (/app-one/.test(acl)) break;
await new Promise((r) => setTimeout(r, 3000));
}
assert.match(acl, /app-one/, `the provisioner never created the consumer's login:\n${(await on(`docker logs mesh-redis 2>&1 | tail -30`)).out}\n---\n${acl}`);
// The proof: authenticate as that consumer with the password the MESH minted. PONG means the
// provisioner created the login with exactly that password. A provisioner that invented its own
// (the old behaviour) would answer WRONGPASS here.
const authed = await on(`docker exec redis redis-cli --user app-one --pass ${quote(password)} --no-auth-warning PING 2>&1`);
assert.doesNotMatch(authed.out, /WRONGPASS/,
`the consumer could not authenticate with the mesh's password — the provider used a different one:\n${authed.out}`);
assert.match(authed.out, /PONG/, `expected PONG authenticating as the consumer:\n${authed.out}`);
// And it needed no seal key: the runtime came up and provisioned with MESH_SEAL_KEY set nowhere.
const env = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`);
assert.doesNotMatch(env, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${env}`);
// The provisioner emitted its lifecycle event under the bound account, and no emit was refused.
const log = (await on(`docker logs mesh-redis 2>&1`)).out;
assert.doesNotMatch(log, /emit .*failed/, `the provisioned event was refused:\n${log}`);
});
+64 -29
View File
@@ -37,7 +37,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent, deriveTheFilterOn } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
@@ -50,14 +50,12 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "route-forwarding"; const SCENARIO = "route-forwarding";
const MACHINE = "anchor"; const MACHINE = "anchor";
const NAME = "hello.example"; const NAME = "hello.example";
const PAGE = "hello from hello-web, routed by the mesh"; const PAGE = "hello from hello-web, routed by the mesh";
/** The node's public domain; hello-web's label composes under it (ADR 0066). */
const DOMAIN = "example";
let instanceId = ""; let instanceId = "";
let held: HeldImage[] = []; let held: HeldImage[] = [];
@@ -87,6 +85,10 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
} }
/** The reference a manifest should carry, once this scenario has been raised. */ /** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */ /** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string { function bundleFor(images: HeldImage[]): string {
@@ -165,31 +167,64 @@ after(async () => {
test("the mesh routes a public name through the proxy to the consumer, and withdraws it on unassign", { test("the mesh routes a public name through the proxy to the consumer, and withdraws it on unassign", {
skip, timeout: 1_500_000, skip, timeout: 1_500_000,
}, async () => { }, async () => {
// All three from the catalogue (novox/hq ADR 0093, issue 074): the authority the proxy requires, // The PROVIDER: route-proxy in the plain-HTTP shape — provides `route`, is given every consumer as
// the proxy, and the consumer. The proxy's manifest names the runtime image the scenario stocks; // the file at receives.route, forwards by Host. No TLS here (that is certificates.test.ts); the
// the authority and the consumer's server are pulled upstream by digest. The name the consumer // image is pinned to what this scenario serves by digest.
// is routed under is composed from its label and the node's public domain (ADR 0066), which const proxyManifest = JSON.stringify({
// the bed sets first. module: "route-proxy",
await mesh(`node public-domain ${MACHINE} ${DOMAIN}`); version: "1",
// The authority certifies itself for the machine's private-network address, which is what a capabilities: ["container-runtime"],
// consumer on any node dials (ADR 0098); a machine raised from the bundle has none until it is provides: [{ name: "route", scope: "mesh" }],
// placed on the overlay. Placed as a hub of one, the way a real first node is, and converged serves: { route: {} },
// BEFORE the modules arrive: the proxy fetches the authority's roots at that address at first receives: { route: "/var/lib/route-proxy/routes/mesh.json" },
// start, so the interface must exist by then — in one push the order between modules is not listens: [{ port: 80, protocol: "tcp", from: "anywhere", why: "public HTTP; the route-forwarding front door" }],
// promised. The derived filter admits the hub's port, as genesis does on a control-node (ADR 0088). resources: [
await mesh(`overlay place ${MACHINE} --hub --endpoint 192.0.2.10:51820 --site lab`); { id: "state", type: "directory", path: "/var/lib/route-proxy", mode: "0700" },
await mesh(`assign ${MACHINE} networking`); { id: "routes-dir", type: "directory", path: "/var/lib/route-proxy/routes", mode: "0700" },
await mesh(`push ${MACHINE}`); {
await settled(); id: "server", type: "container", name: "route-proxy",
await deriveTheFilterOn({ machine: MACHINE, node: MACHINE, hubPort: 51820, image: pinned("mesh-route-proxy"), network: "host",
must: (_m, c, t) => must(c, t), mesh, on: (_m, c, t) => on(c, t) }); volumes: ["/var/lib/route-proxy/routes:/routes:ro"],
const overlay = await must(`ip -4 addr show dev mesh0 2>&1 || ip -4 addr 2>&1`); env: { ROUTES: "/routes/mesh.json", LISTEN: ":80" },
assert.match(overlay, /inet 10\./, `${MACHINE} has no private-network address after networking converged:\n${overlay}`); },
for (const name of ["step-ca", "route-proxy", "hello-web"]) { ],
await must(`printf %s ${quote(catalogueModule(name, held))} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); });
await mesh(`module add /${name}.json`);
await mesh(`assign ${MACHINE} ${name}`); // The CONSUMER: hello-web requires `route` and contributes the name it wants and the port it
} // listens on. It runs no code of the mesh's — a bare alpine serving a fixed page over a busybox nc
// loop stands in for a web service. `contributes` is what makes it *ask*: the grant forms from it.
const webManifest = JSON.stringify({
module: "hello-web",
slug: "hello",
version: "1",
capabilities: ["container-runtime"],
requires: ["route"],
contributes: { route: { name: NAME, port: 8080 } },
binds: { route: "/var/lib/hello-web/route.json" },
listens: [{ port: 8080, protocol: "tcp", from: "mesh", why: "the demo page; only the proxy reaches it" }],
resources: [
{ id: "state", type: "directory", path: "/var/lib/hello-web", mode: "0700" },
{ id: "page", type: "file", path: "/var/lib/hello-web/index.html", mode: "0644", content: `${PAGE}\n` },
{ id: "net", type: "network", name: "hello-web" },
{
id: "server", type: "container", name: "hello-web",
image: pinned("alpine"), network: "hello-web", ports: ["8080:8080"],
volumes: ["/var/lib/hello-web/index.html:/www/index.html:ro"],
args: ["sh", "-c",
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done"],
},
],
});
await must(`printf %s ${quote(proxyManifest)} > /tmp/route-proxy.json && docker cp /tmp/route-proxy.json mesh-controller:/route-proxy.json`);
await mesh("module add /route-proxy.json");
// No `module issue`: route-proxy has no broker account and no own-secret to mint. `assign` resolves
// its plan and the provider is matchable by a consumer's route from that alone.
await mesh(`assign ${MACHINE} route-proxy`);
await must(`printf %s ${quote(webManifest)} > /tmp/hello-web.json && docker cp /tmp/hello-web.json mesh-controller:/hello-web.json`);
await mesh("module add /hello-web.json");
await mesh(`assign ${MACHINE} hello-web`);
await mesh(`push ${MACHINE}`); await mesh(`push ${MACHINE}`);
await settled(); await settled();
@@ -8,7 +8,7 @@
* service has: the runtime names its config resource, and the host recreates the container when that * service has: the runtime names its config resource, and the host recreates the container when that
* resource changed this pass. * resource changed this pass.
* *
* This assigns a settings-configured runtime (the fixture wears no catalogue name; its image is grafana's tool runtime), then changes the token and pushes again, and asserts * This assigns grafana configured by settings, then changes the token and pushes again, and asserts
* the container was replaced (a new container id) and the config on disk carries the new value. * the container was replaced (a new container id) and the config on disk carries the new value.
* It builds the host from source (no --no-build), because the behaviour under test is the host's. * It builds the host from source (no --no-build), because the behaviour under test is the host's.
* *
@@ -113,7 +113,7 @@ async function settled(withinMs = 480_000): Promise<void> {
async function setToken(token: string): Promise<void> { async function setToken(token: string): Promise<void> {
const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token }); const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token });
await must(`printf %s ${quote(settings)} > /tmp/s.json && docker cp /tmp/s.json mesh-controller:/s.json`); await must(`printf %s ${quote(settings)} > /tmp/s.json && docker cp /tmp/s.json mesh-controller:/s.json`);
await mesh(`settings set a-runtime /s.json --node ${MACHINE}`); await mesh(`settings set grafana /s.json --node ${MACHINE}`);
} }
async function containerId(): Promise<string> { async function containerId(): Promise<string> {
@@ -151,7 +151,7 @@ test("a running runtime is recreated when its settings change, and reads the new
skip, timeout: 900_000, skip, timeout: 900_000,
}, async () => { }, async () => {
const manifest = JSON.stringify({ const manifest = JSON.stringify({
module: "a-runtime", module: "grafana",
version: "1", version: "1",
emits: ["module.grafana.alert.firing"], emits: ["module.grafana.alert.firing"],
"own-secrets": { broker: "/var/lib/mesh/grafana/broker" }, "own-secrets": { broker: "/var/lib/mesh/grafana/broker" },
@@ -170,12 +170,12 @@ test("a running runtime is recreated when its settings change, and reads the new
}, },
], ],
}); });
await must(`printf %s ${quote(manifest)} > /tmp/a-runtime.json && docker cp /tmp/a-runtime.json mesh-controller:/a-runtime.json`); await must(`printf %s ${quote(manifest)} > /tmp/grafana.json && docker cp /tmp/grafana.json mesh-controller:/grafana.json`);
await mesh("module add /a-runtime.json"); await mesh("module add /grafana.json");
await setToken("token-alpha"); await setToken("token-alpha");
await mesh(`module issue a-runtime --node ${MACHINE}`); await mesh(`module issue grafana --node ${MACHINE}`);
await mesh(`assign ${MACHINE} a-runtime`); await mesh(`assign ${MACHINE} grafana`);
await mesh(`push ${MACHINE}`); await mesh(`push ${MACHINE}`);
await settled(); await settled();
-198
View File
@@ -1,198 +0,0 @@
/**
* **Nothing a machine sends while the store restarts is lost** (novox/hq issues 082, 083).
*
* The foundation's store is recreated when it is adopted, and for those seconds the control plane
* cannot write. This bed takes the store away on the control-node, has a second machine enrol into
* the gap, and brings the store back after the control plane has had to say "not now":
*
* - a report arriving in the gap is held, and recorded when the store is back;
* - the enrolment is answered "not now" while that report is held — not queued behind it — and
* completes on its own when the store is back, with the keys it started with: the mesh holds
* the very keys the machine generated;
* - the machine then applies what it is pushed and is heard from.
*
* It needs a host binary and the foundation bundle:
*
* MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary)
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
const SCENARIO = "store-window";
let instanceId = "";
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, machine, [
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(machine, command, timeoutMs);
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}
function tokenFrom(said: string): string {
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
assert.ok(found, `no token in:\n${said}`);
return found;
}
async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
const end = Date.now() + within;
while (Date.now() < end) {
if (await check()) return;
await new Promise((r) => setTimeout(r, 3000));
}
assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
held = raised.images;
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
}
await mesh("node add anchor");
const own = tokenFrom(await mesh("token issue --node anchor"));
await must("anchor", `${HOST_PATH} enrol --token ${quote(own)}`);
await must("anchor", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });
after(async () => {
if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
test("a machine enrolling while the store is away joins when it comes back, with the keys it started with", {
skip, timeout: 900_000,
}, async () => {
await mesh("node add laptop");
const token = tokenFrom(await mesh("token issue --node laptop"));
// A report that will arrive in the gap: the anchor is pushed a step that takes ten seconds, so
// its report lands after the store has gone.
const sleeper = onTheMachine("alpine", held);
await must("anchor", `printf %s '{"module":"slow","version":"1","resources":[` +
`{"id":"step","type":"container","name":"slow-step","image":"${sleeper}","run-once":true,` +
`"args":["sh","-c","sleep 10"]}]}' > /tmp/slow.json`);
await must("anchor", `docker cp /tmp/slow.json mesh-controller:/slow.json`);
await mesh("module add /slow.json");
await mesh("assign anchor slow");
await mesh("push anchor");
// The store goes away, as it does when the foundation is adopted; the broker stays, so the
// report and the enrolment reach the control plane and the control plane cannot write.
await must("anchor", `docker stop mesh-store`);
await until("the anchor's report arriving in the gap and being held", 120_000,
async () => /could not keep anchor's report .*holding it/.test((await on("anchor", `docker logs mesh-controller 2>&1`)).out),
async () => `--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}\n` +
`--- anchor host ---\n${(await on("anchor", `tail -10 /var/log/mesh-host.log`)).out}`);
// The machine enrols into the gap. In the background: it will be told "not now" and keep asking.
await must("laptop", `nohup sh -c ${quote(`${HOST_PATH} enrol --token ${quote(token)} > /tmp/enrol.log 2>&1; ` +
`echo "exit=$?" >> /tmp/enrol.log`)} > /dev/null 2>&1 & sleep 1`);
// The control plane said "not now" at least once, while the anchor's report was held — so the
// gap was hit, and the enrolment was answered rather than queued behind what the store owed.
await until("the control plane asking the machine to enrol again", 90_000,
async () => /asked "laptop" to enrol again/.test((await on("anchor", `docker logs mesh-controller 2>&1`)).out),
async () => `--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}\n` +
`--- laptop enrol ---\n${(await on("laptop", `cat /tmp/enrol.log`)).out}`);
// The store comes back; the enrolment completes on its own, with nothing done by hand.
await must("anchor", `docker start mesh-store`);
await until("the enrolment completing", 150_000,
async () => /exit=/.test((await on("laptop", `cat /tmp/enrol.log`)).out),
async () => `--- laptop enrol ---\n${(await on("laptop", `cat /tmp/enrol.log`)).out}\n` +
`--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}`);
const said = (await on("laptop", `cat /tmp/enrol.log`)).out;
assert.match(said, /exit=0/, `the enrolment did not complete after the store came back:\n${said}`);
assert.match(said, /enrolled as laptop/, `the machine was not enrolled as laptop:\n${said}`);
// The mesh holds the very keys the machine generated at the start: its identity is the live key,
// and its sealing key is the one on its record.
const identity = said.match(/generated this node's identity: (\S+)/)?.[1];
const sealing = said.match(/generated this node's sealing key:\s+(\S+)/)?.[1];
assert.ok(identity && sealing, `the enrolment did not say which keys it generated:\n${said}`);
const nodeId = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
`-c "select id from node where name = 'laptop'"`)).trim();
const live = (await must("anchor", `docker exec mesh-store psql -U postgres -d identity -qAt ` +
`-c "select encode(public, 'base64') from node_key where node = '${nodeId}' and revoked is null"`)).trim();
assert.equal(live, identity, `the mesh's live key for laptop is not the one it generated`);
const sealedTo = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
`-c "select sealing_key from node where name = 'laptop'"`)).trim();
assert.equal(sealedTo, sealing, `the mesh's sealing key for laptop is not the one it generated`);
// The report held through the gap was recorded once the store was back.
await until("the anchor's held report being recorded", 60_000,
async () => {
const r = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
if (!r.ok) return false;
const state = JSON.parse(r.out) as { reported: { node: string; outcome: string; current: boolean }[] };
return state.reported.some((w) => w.node === "anchor" && w.outcome === "applied" && w.current);
},
async () => (await on("anchor", `docker logs --tail 20 mesh-controller 2>&1`)).out);
// And the machine is a working member: pushed something, it applies it and is heard from.
await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
await must("anchor", `printf %s '{"module":"marker","version":"1","resources":[` +
`{"id":"marker","type":"file","path":"/etc/store-window","content":"joined\\\\n","mode":"0644"}]}' > /tmp/marker.json`);
await must("anchor", `docker cp /tmp/marker.json mesh-controller:/marker.json`);
await mesh("module add /marker.json");
await mesh("assign laptop marker");
await mesh("push laptop");
await until("the machine applying what it was pushed", 120_000,
async () => (await on("laptop", `grep -q joined /etc/store-window`)).ok,
async () => (await on("laptop", `tail -20 /var/log/mesh-host.log`)).out);
await until("the mesh hearing the machine's report", 120_000,
async () => {
const r = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
if (!r.ok) return false;
const state = JSON.parse(r.out) as { reported: { node: string; outcome: string; current: boolean }[] };
return state.reported.some((w) => w.node === "laptop" && w.outcome === "applied" && w.current);
},
async () => (await on("anchor", `docker exec mesh-controller /mesh-controller status 2>&1`)).out);
});
+26 -11
View File
@@ -4,7 +4,7 @@
* whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set. * whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set.
* *
* Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the * Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the
* `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres * `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
* providers co-located with them. The media stack shares the operator-owned library directories * providers co-located with them. The media stack shares the operator-owned library directories
* under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS * under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS
* each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those * each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those
@@ -25,17 +25,19 @@
import { test, before, after } from "node:test"; import { test, before, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync } from "node:fs"; import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts"; import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, deriveTheFilterOn, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
const binary = hostBinaryPath(); const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
const skip = !capability.usable const skip = !capability.usable
? `lab not usable: ${capability.why}` ? `lab not usable: ${capability.why}`
@@ -43,12 +45,14 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "whole-mesh-ace"; const SCENARIO = "whole-mesh-ace";
const NODE = "ace"; const NODE = "ace";
const catalogDir = process.env["MESH_LAB_CATALOG"]
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
/** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */ /** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */
const MEDIA_DIRS = [ const MEDIA_DIRS = [
@@ -171,17 +175,27 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
} }
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string { function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images); return foundationBundle(bundle, images);
} }
/** The catalogue's manifest as the lab runs it (harness). This bed's own loader never resolved a
* runtime ARTIFACT to a stocked image, so every module whose runtime the mesh builds travelled to
* the machine unresolved — the shared loader does (novox/hq 04-ISSUES/073). */
function loadManifest(name: string): { manifest: string; broker: boolean } { function loadManifest(name: string): { manifest: string; broker: boolean } {
const manifest = catalogueModule(name, held, { ports: REMAP[name] }); const path = resolve(catalogDir, name, "module.json");
return { manifest, broker: needsBrokerAccount(manifest) }; const m = JSON.parse(readFileSync(path, "utf8")) as {
resources?: { type: string; image?: string; ports?: string[] }[];
};
const remap = REMAP[name] ?? {};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
if (typeof r.image === "string") r.image = pinned(r.image);
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
}
const manifest = JSON.stringify(m);
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
} }
function tokenFrom(said: string): string { function tokenFrom(said: string): string {
@@ -226,6 +240,7 @@ async function nodeState(node: string): Promise<NodeState> {
before(async () => { before(async () => {
if (skip) return; if (skip) return;
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`), onProgress: (m) => console.log(`raise: ${m}`),
@@ -264,7 +279,7 @@ test("the whole ace service set resolves, installs and converges on one node in
}, async () => { }, async () => {
for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`); for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
// The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres). // The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres/redis).
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
await mesh(`overlay place ${NODE} --site lab`); await mesh(`overlay place ${NODE} --site lab`);
await mesh("assign anchor networking"); await mesh("assign anchor networking");
+106 -24
View File
@@ -58,20 +58,21 @@ import { test, before, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join, resolve } from "node:path"; import { dirname, join, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts"; import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts"; import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts";
import { import {
bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH, bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH,
} from "../../src/lifecycle/place.ts"; } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, catalogueDir, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import { referenceFor, type HeldImage } from "../../src/pinning.ts"; import { referenceFor, type HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
const binary = hostBinaryPath(); const binary = hostBinaryPath();
const installer = bootstrapBinaryPath(); const installer = bootstrapBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
// What the installer is told to build. It carries a builder rather than a finished control plane // What the installer is told to build. It carries a builder rather than a finished control plane
// (novox/hq ADR 0073), so genesis is given a repository and a commit — and a commit rather than a // (novox/hq ADR 0073), so genesis is given a repository and a commit — and a commit rather than a
// branch, because what is cloned is the trust anchor for everything this mesh will ever run. // branch, because what is cloned is the trust anchor for everything this mesh will ever run.
@@ -92,7 +93,7 @@ const skip = !capability.usable
? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from"
: !sourceRef : !sourceRef
? "MESH_LAB_SOURCE_REF is not set to the commit to build" ? "MESH_LAB_SOURCE_REF is not set to the commit to build"
: catalogueIsPresent(); : false;
const SCENARIO = "whole-mesh-full"; const SCENARIO = "whole-mesh-full";
/** novox hosts the foundation and the control plane; it is where `mesh` commands run. */ /** novox hosts the foundation and the control plane; it is where `mesh` commands run. */
@@ -153,8 +154,9 @@ const PUBLIC_DOMAIN: Record<string, string> = { novox: "novox.incus", ace: "zura
const KEEP = !!process.env["MESH_LAB_KEEP"]; const KEEP = !!process.env["MESH_LAB_KEEP"];
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined); const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined);
/** The catalogue's modules directory (harness). Absent, the bed skips — see `skip`. */ const catalogDir = process.env["MESH_LAB_CATALOG"]
const catalogDir = catalogueIsPresent() ? "" : catalogueDir(); ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
const MEDIA_DIRS = [ const MEDIA_DIRS = [
"/services/media/series", "/services/media/anime", "/services/media/movies", "/services/media/series", "/services/media/anime", "/services/media/movies",
@@ -179,9 +181,6 @@ const NOVOX: Mod[] = [
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, { name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
{ name: "mssql", containers: ["mssql", "mesh-mssql"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] },
{ name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] }, { name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] },
// The vault, before everything that keeps a secret from it: gitea, umami, influxdb, mailu
// require one (novox/hq ADRs 0085, 0094).
{ name: "mesh-vault", containers: ["mesh-vault"] },
// ADR 0066: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or // ADR 0066: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or
// route-proxy is unresolvable and takes every routed module down with it. step-ca is that // route-proxy is unresolvable and takes every routed module down with it. step-ca is that
// provider, on the anchor, at mesh scope. // provider, on the anchor, at mesh scope.
@@ -204,7 +203,7 @@ const NOVOX: Mod[] = [
// what an operator's `module add` + `assign` would do on a mesh that already has it, and a // what an operator's `module add` + `assign` would do on a mesh that already has it, and a
// module the installer put there had better survive being asked for a second time. It also keeps // module the installer put there had better survive being asked for a second time. It also keeps
// mesh-registry in the convergence report, where a reader expects to see it. // mesh-registry in the convergence report, where a reader expects to see it.
{ name: "distribution", containers: ["mesh-registry"] }, { name: "registry", containers: ["mesh-registry"] },
{ {
name: "mailu", name: "mailu",
containers: [ containers: [
@@ -213,16 +212,16 @@ const NOVOX: Mod[] = [
"mailu-front", "mesh-mailu", "mailu-front", "mesh-mailu",
], ],
}, },
{ name: "nftables", containers: [], node: true }, { name: "firewall", containers: [], node: true },
{ name: "fail2ban", containers: [], node: true }, { name: "fail2ban", containers: [], node: true },
]; ];
const CORE_NOVOX = new Set([ const CORE_NOVOX = new Set([
"postgres", "redis", "minio", "mongodb", "mssql", "lavinmq", "postgres", "redis", "minio", "mongodb", "mssql", "lavinmq",
"route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be", "route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be",
"portainer", "verdaccio", "distribution", "portainer", "verdaccio", "registry",
]); ]);
const GAPS_NOVOX = new Set([ const GAPS_NOVOX = new Set([
"umami", "mailu", "nftables", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder",
// step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in // step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in
// mesh-controller, and this bed is not the place to discover that a fix has not landed yet. What is // mesh-controller, and this bed is not the place to discover that a fix has not landed yet. What is
// gated is the half that is decided and cheap — see the ADR 0066 section at the end. // gated is the half that is decided and cheap — see the ADR 0066 section at the end.
@@ -309,15 +308,18 @@ const CREDENTIALS: { node: string; module: string; name: string; crash: string }
{ node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" }, { node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" },
{ node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" }, { node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" },
{ node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" }, { node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" },
{ node: "novox", module: "umami", name: "admin", crash: "admin password is not set" },
]; ];
/** Operator secrets for the credential modules that own-secret their whole app (de-spiegel, /** Operator secrets for the credential modules that own-secret their whole app (mailu, de-spiegel). */
* amqp-email-forwarder). mailu's and umami's are kept in the vault now (ADR 0094), not delivered. */
const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [ const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [
{ node: "novox", module: "mailu", name: "secret-key", value: "0123456789abcdef0123456789abcdef" },
{ node: "novox", module: "mailu", name: "admin", value: "MailuAdminFakePass123" },
{ node: "novox", module: "mailu", name: "api-token", value: "mailuapitokenfake0123456789abcd" },
{ node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" }, { node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" },
{ node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" }, { node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" },
{ node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" }, { node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" },
{ node: "novox", module: "amqp-email-forwarder", name: "smtp-password", value: "eef-pass-fake" }, { node: "novox", module: "amqp-email-forwarder", name: "smtp-pass", value: "eef-pass-fake" },
]; ];
let instanceId = ""; let instanceId = "";
@@ -348,16 +350,45 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
} }
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: HeldImage[]): string { function bundleFor(images: HeldImage[]): string {
return foundationBundle(bundle, images); return foundationBundle(bundle, images);
} }
/** The catalogue's manifest as the lab runs it (harness): artifacts resolved to the images the
* scenario stocked — the lab standing in for the builder — images pinned, host ports remapped. */
function loadManifest(name: string): { manifest: string; broker: boolean } { function loadManifest(name: string): { manifest: string; broker: boolean } {
const manifest = catalogueModule(name, held, { ports: REMAP[name] }); const path = resolve(catalogDir, name, "module.json");
return { manifest, broker: needsBrokerAccount(manifest) }; const m = JSON.parse(readFileSync(path, "utf8")) as {
resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[];
};
const remap = REMAP[name] ?? {};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
// **A container naming an artifact is a module the mesh builds, and this bed does not build.**
// It pre-builds the same images on the workstation and stocks them, which is the lab standing
// in for the builder — so it does here what the builder does: replace the artifact with the
// reference the machine actually holds. Without this the unresolved field travels to the
// machine, whose declaration language has no such field, and the whole declaration is refused.
//
// The repository is `mesh-runtime-<module>`, which is not a guess: it is what this repository's
// own `scripts/build-module-runtime.sh <module>` produces and what the scenarios stock by name.
if (typeof r.artifact === "string" && typeof r.image !== "string") {
const reference = referenceFor(held, `mesh-runtime-${name}`);
assert.ok(reference,
`${name} declares the "${r.artifact}" artifact and this scenario stocked no ` +
`mesh-runtime-${name}. The mesh would have to build it, and this bed does not build — ` +
`add it to the machine's images: in the scenario, or build it with ` +
`scripts/build-module-runtime.sh ${name}`);
r.image = reference;
delete r.artifact;
}
if (typeof r.image === "string") r.image = pinned(r.image);
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
}
const manifest = JSON.stringify(m);
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
} }
function tokenFrom(said: string): string { function tokenFrom(said: string): string {
@@ -410,8 +441,56 @@ async function psMapOf(node: string): Promise<Map<string, string>> {
return map; return map;
} }
// ADR 0098: the internal authority makes its own root at first start and serves it; the proxy /**
// fetches it. No operator root is delivered — the mesh mints only the authority's password. * ADR 0066: the internal CA is initialised FROM AN OPERATOR'S ROOT — it does not mint its own.
*
* So the bed has to be an operator. The material is made on the anchor with openssl and handed to
* the mesh through the real `secret accept` path, exactly as a person would: the mesh cannot invent
* a PEM, and the random 32 bytes it makes for an own-secret nobody supplied would leave step-ca
* crash-looping on a root key that is not a key.
*/
async function deliverCaRoot(): Promise<boolean> {
const made = await on(CONTROL, [
"set -e",
"mkdir -p /tmp/ca && cd /tmp/ca",
// No trailing newline on a password file: step-ca reads the file as the password itself.
"openssl rand -hex 16 | tr -d '\\n' > key-password",
"openssl ecparam -genkey -name prime256v1 -out root.unenc",
"openssl ec -in root.unenc -aes256 -passout file:key-password -out root.key",
"rm -f root.unenc",
"openssl req -x509 -new -key root.key -passin file:key-password -sha256 -days 3650" +
` -out root.crt -subj "/CN=Mesh Internal CA/O=Novox Mesh Lab"`,
// Readable by the control plane, which is not root. Its image is FROM scratch and runs as
// 65534, and `docker cp` keeps the ownership and mode a file had outside — openssl writes a
// private key 0600 root-owned, so the copy landed unreadable and `secret accept` failed with
// `open /ca-root-key: permission denied`. The CA then crash-looped on a root it never got.
// Chowning it inside the container is not available: there is no shell in there to do it with.
//
// Safe here and nowhere else: these three exist for the seconds between being written and
// being sealed to the machine, on a lab node, for a CA thrown away with the scenario.
"chmod 0644 /tmp/ca/root.crt /tmp/ca/root.key /tmp/ca/key-password",
"docker cp /tmp/ca/root.crt mesh-controller:/ca-root-cert",
"docker cp /tmp/ca/root.key mesh-controller:/ca-root-key",
"docker cp /tmp/ca/key-password mesh-controller:/ca-root-key-password",
].join("\n"), 180_000);
if (!made.ok) {
console.log(`CA ROOT NOT MADE on ${CONTROL}:\n${made.out.split("\n").slice(-8).join("\n")}`);
return false;
}
for (const [name, file] of [
["root-cert", "/ca-root-cert"],
["root-key", "/ca-root-key"],
["root-key-password", "/ca-root-key-password"],
] as const) {
try {
await mesh(`secret accept ${CONTROL} step-ca ${name} --from ${file}`);
} catch (err) {
console.log(`CA ROOT ACCEPT FAILED (${name}): ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`);
return false;
}
}
return true;
}
/** What a module's manifest says its route label is, or "" if it contributes no route. */ /** What a module's manifest says its route label is, or "" if it contributes no route. */
function routeLabelOf(name: string): string { function routeLabelOf(name: string): string {
@@ -668,6 +747,7 @@ async function joinTheMesh(): Promise<void> {
before(async () => { before(async () => {
if (skip) return; if (skip) return;
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`), onProgress: (m) => console.log(`raise: ${m}`),
@@ -800,9 +880,6 @@ test("the full mesh forms across the access point and both server sets converge"
for (const { name } of mods) { for (const { name } of mods) {
try { try {
const broker = await ensureAdded(name); const broker = await ensureAdded(name);
// Issued only when the module holds a broker account: an own secret the mesh mints
// (step-ca's password) is minted at resolve, and `module issue` refuses a module with no
// broker secret to deliver into (issue 078).
if (broker) await mesh(`module issue ${name} --node ${node}`); if (broker) await mesh(`module issue ${name} --node ${node}`);
await mesh(`assign ${node} ${name}`); await mesh(`assign ${node} ${name}`);
assigned[node]!.add(name); assigned[node]!.add(name);
@@ -844,6 +921,10 @@ test("the full mesh forms across the access point and both server sets converge"
} }
} }
// ADR 0066: the CA's root, before the push that would otherwise deliver a random 32 bytes for it.
const caRootDelivered = assigned["novox"]!.has("step-ca") ? await deliverCaRoot() : false;
if (!caRootDelivered) console.log("ADR 0066: no operator root delivered; step-ca cannot initialise.");
// ONE push per node (workstations first — cheap — then the heavy service nodes). // ONE push per node (workstations first — cheap — then the heavy service nodes).
const pushError: Record<string, string> = {}; const pushError: Record<string, string> = {};
for (const node of ["shanks", "g14", "novox", "ace"]) { for (const node of ["shanks", "g14", "novox", "ace"]) {
@@ -978,6 +1059,7 @@ test("the full mesh forms across the access point and both server sets converge"
? (await on("novox", `curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H 'Host: ${probe.name}' http://127.0.0.1/`)).out.trim() ? (await on("novox", `curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H 'Host: ${probe.name}' http://127.0.0.1/`)).out.trim()
: ""; : "";
adr.push(`\n proxy answers for ${probe?.name ?? "(nothing composed)"}: ${servedCode || "no answer"}`); adr.push(`\n proxy answers for ${probe?.name ?? "(nothing composed)"}: ${servedCode || "no answer"}`);
adr.push(` operator root delivered to step-ca: ${caRootDelivered}`);
adr.push(` step-ca container: ${psMaps["novox"]?.get("step-ca") ?? "MISSING"}`); adr.push(` step-ca container: ${psMaps["novox"]?.get("step-ca") ?? "MISSING"}`);
console.log(adr.join("\n")); console.log(adr.join("\n"));
+40 -13
View File
@@ -36,17 +36,19 @@
import { test, before, after } from "node:test"; import { test, before, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync } from "node:fs"; import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts"; import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts"; import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts"; import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable(); const capability = await labIsUsable();
const binary = hostBinaryPath(); const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
const skip = !capability.usable const skip = !capability.usable
? `lab not usable: ${capability.why}` ? `lab not usable: ${capability.why}`
@@ -54,12 +56,15 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) : !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: catalogueIsPresent(); : false;
const SCENARIO = "whole-mesh-novox"; const SCENARIO = "whole-mesh-novox";
const NODE = "novox"; const NODE = "novox";
/** Where the committed module.json files live: the mesh-catalog beside mesh-controller. */
const catalogDir = process.env["MESH_LAB_CATALOG"]
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
/** /**
* The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and * The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and
@@ -68,9 +73,6 @@ const NODE = "novox";
*/ */
const MODULES: { name: string; containers: string[]; node?: boolean }[] = [ const MODULES: { name: string; containers: string[]; node?: boolean }[] = [
{ name: "postgres", containers: ["mesh-store", "mesh-postgres"] }, { name: "postgres", containers: ["mesh-store", "mesh-postgres"] },
// The vault, before everything that keeps a secret from it: redis, gitea, umami, influxdb,
// mailu require one (novox/hq ADRs 0085, 0094).
{ name: "mesh-vault", containers: ["mesh-vault"] },
{ name: "redis", containers: ["redis", "mesh-redis"] }, { name: "redis", containers: ["redis", "mesh-redis"] },
{ name: "minio", containers: ["minio", "mesh-minio"] }, { name: "minio", containers: ["minio", "mesh-minio"] },
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, { name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
@@ -227,14 +229,38 @@ function bundleFor(images: HeldImage[]): string {
} }
/** /**
* The catalogue's manifest as the lab runs it (harness): images pinned, artifacts resolved to the * Load a committed module.json, rewrite every container image to the scenario's pinned digest, and
* stocked images, the host-port remaps applied. Returns the manifest and whether it needs a broker * apply the host-port remaps. Returns the manifest as a string and whether it needs a broker account
* account (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules * (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules do not).
* do not).
*/ */
function loadManifest(name: string): { manifest: string; broker: boolean } { function loadManifest(name: string): { manifest: string; broker: boolean } {
const manifest = catalogueModule(name, held, { ports: REMAP[name] }); const path = resolve(catalogDir, name, "module.json");
return { manifest, broker: needsBrokerAccount(manifest) }; const m = JSON.parse(readFileSync(path, "utf8")) as {
resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[];
build?: unknown;
};
const remap = REMAP[name] ?? {};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
if (typeof r.image === "string") {
r.image = pinned(r.image);
} else if (typeof r.artifact === "string") {
// Since issue 060 a module's own runtime container names an artifact the mesh's builder
// would fill, not a placeholder image. This bed stocks the image instead of building, so
// map the artifact to the stocked `mesh-runtime-<module>` the machine holds — keyed on the
// MODULE name, not the container's (mailu's runtime container is `mesh-mailu`, its image is
// `mesh-runtime-mailu`). The placeholder digest is what `pinned` already resolves for a
// mesh-built repo, exactly as it did for the old `image` field.
r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`);
delete r.artifact;
}
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
}
// The build section the mesh's builder would consume: dropped, because this bed stocks the image
// rather than building it. Harmless to leave (the push path never reads it), removed for clarity.
delete m.build;
const manifest = JSON.stringify(m);
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
} }
function tokenFrom(said: string): string { function tokenFrom(said: string): string {
@@ -280,6 +306,7 @@ async function nodeState(node: string): Promise<NodeState> {
before(async () => { before(async () => {
if (skip) return; if (skip) return;
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
onProgress: (m) => console.log(`raise: ${m}`), onProgress: (m) => console.log(`raise: ${m}`),
-16
View File
@@ -104,28 +104,12 @@ test("every repository the receipt claims was built by the run", () => {
MESH_LAB_HOST_BINARY: "/repo/host/mesh-host", MESH_LAB_HOST_BINARY: "/repo/host/mesh-host",
MESH_LAB_MODULES: "/repo/control/examples/modules", MESH_LAB_MODULES: "/repo/control/examples/modules",
MESH_LAB_BUILDER: "/repo/control/build/mesh-builder", MESH_LAB_BUILDER: "/repo/control/build/mesh-builder",
MESH_LAB_CATALOG: "/repo/catalog/modules",
}; };
const built = new Set(planned(env).map((b) => b.in)); const built = new Set(planned(env).map((b) => b.in));
for (const [name, directory] of Object.entries(repositories(env))) { for (const [name, directory] of Object.entries(repositories(env))) {
// mesh-lab is the exception, and it is not an omission: it is TypeScript run from source, so // mesh-lab is the exception, and it is not an omission: it is TypeScript run from source, so
// the code under test *is* the code running. There is nothing to build and nothing to go stale. // the code under test *is* the code running. There is nothing to build and nothing to go stale.
if (name === "mesh-lab") continue; if (name === "mesh-lab") continue;
// mesh-catalog is the other exception, for the other reason: what a bed takes from it is a
// manifest, read from disk when the bed runs. There is nothing built from it that could go
// stale — and claiming it is the whole point, since a bed that carried its own copy of the
// manifest was proving the copy (novox/hq 04-ISSUES/073).
if (name === "mesh-catalog") continue;
assert.ok(built.has(directory), `${name} (${directory}) is claimed but never built`); assert.ok(built.has(directory), `${name} (${directory}) is claimed but never built`);
} }
}); });
// The catalogue is claimed by the receipt, under either spelling the beds accept.
//
// A bed reads the manifest it installs from the catalogue checkout (novox/hq 04-ISSUES/073), so a
// receipt that names no catalogue commit cannot say whether a change there was ever proven.
test("the receipt claims the catalogue the beds read, however it was named", () => {
assert.equal(repositories({ MESH_LAB_CATALOG: "/repo/catalog/modules" })["mesh-catalog"], "/repo/catalog");
assert.equal(repositories({ MESH_LAB_CATALOG: "/repo/catalog" })["mesh-catalog"], "/repo/catalog");
assert.equal(repositories({})["mesh-catalog"], undefined);
});
-88
View File
@@ -1,88 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { ageOfImage, ageOfSource, isStale, plannedRuntimes, runtimesStockedBy, type Ask } from "../src/runtimes.ts";
// The module runtimes a run stocks are rebuilt by the run (novox/hq 04-ISSUES/075): what a bed's
// scenario stocks is found, compared against its source, and built where older.
function aLabWith(beds: Record<string, string>, scenarios: Record<string, string[]>): { root: string; done: () => void } {
const root = mkdtempSync(join(tmpdir(), "mesh-lab-runtimes-"));
mkdirSync(join(root, "scenarios"));
mkdirSync(join(root, "test", "integration"), { recursive: true });
for (const [name, images] of Object.entries(scenarios)) {
writeFileSync(join(root, "scenarios", `${name}.yml`), `scenario: ${name}\nimages:\n${images.map((i) => ` - ${i}\n`).join("")}`);
}
for (const [file, scenario] of Object.entries(beds)) {
writeFileSync(join(root, "test", "integration", file), `const SCENARIO = "${scenario}";\n`);
}
return { root, done: () => rmSync(root, { recursive: true, force: true }) };
}
test("what a bed's scenario stocks is found, by module, once", () => {
const lab = aLabWith(
{ "a.test.ts": "one", "b.test.ts": "two", "c.test.ts": "one" },
{ one: ["mesh-controller:development", "mesh-runtime-gitlab:development", "postgres:16"],
two: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
try {
const found = runtimesStockedBy(["test/integration/a.test.ts", "test/integration/b.test.ts", "test/integration/c.test.ts"], lab.root);
assert.deepEqual(found, [
{ tag: "mesh-runtime-gitlab:development", module: "gitlab" },
// The audit logger's runtime is stocked under its slug, and the module is named for it.
{ tag: "mesh-runtime-audit:development", module: "audit-logger" },
]);
} finally { lab.done(); }
});
test("an image is stale when missing, older than its source, or when the source is uncommitted", () => {
assert.equal(isStale({ image: null, source: 0 }), true);
assert.equal(isStale({ image: 100, source: 200 }), true);
assert.equal(isStale({ image: 200, source: 100 }), false);
assert.equal(isStale({ image: 200, source: Infinity }), true);
});
test("the image's age comes from the store and the source's from the newest commit in any part", () => {
const answers: Ask = (command, args) => {
if (command === "docker") return { status: 0, stdout: "2026-09-21T12:00:00.000000000Z\n" };
if (args.includes("status")) return { status: 0, stdout: "" };
if (args.includes("modules/gitlab")) return { status: 0, stdout: "1000\n" };
return { status: 0, stdout: args[1] === "/tools" ? "3000\n" : "2000\n" };
};
assert.equal(ageOfImage("mesh-runtime-gitlab:development", answers), Date.parse("2026-09-21T12:00:00Z") / 1000);
assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], answers), 3000);
// No such image is null, not zero: "never built" and "built at the epoch" are different answers.
assert.equal(ageOfImage("mesh-runtime-nothing:development", () => ({ status: 1, stdout: "" })), null);
// Uncommitted changes anywhere in the source are newer than every commit.
const dirtyTools: Ask = (command, args) =>
args.includes("status") && args[1] === "/tools" ? { status: 0, stdout: " M src/x.ts\n" } : answers(command, args);
assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], dirtyTools), Infinity);
});
test("only a stale runtime is planned, built by the lab's own script under the tag the scenario stocks", () => {
const lab = aLabWith({ "a.test.ts": "one" },
{ one: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
try {
const answers: Ask = (command, args) => {
if (command === "docker") {
// gitlab's image is from yesterday; the audit logger has none.
return args.includes("mesh-runtime-gitlab:development")
? { status: 0, stdout: "2026-09-21T12:00:00Z\n" } : { status: 1, stdout: "" };
}
if (args.includes("status")) return { status: 0, stdout: "" };
return { status: 0, stdout: `${Date.parse("2026-09-20T00:00:00Z") / 1000}\n` };
};
const env = { MESH_LAB_CATALOG: "/catalogue/modules", MESH_TOOLS: "/tools", MESH_SDK: "/sdk" };
const builds = plannedRuntimes(["test/integration/a.test.ts"], env, lab.root, answers);
assert.equal(builds.length, 1);
assert.equal(builds[0]!.what, "runtime mesh-runtime-audit:development");
assert.equal(builds[0]!.argv[0], "scripts/build-module-runtime.sh");
assert.equal(builds[0]!.argv[1], "audit-logger");
assert.equal(builds[0]!.env?.["MESH_CATALOG"], "/catalogue");
assert.equal(builds[0]!.env?.["RUNTIME_TAG"], "mesh-runtime-audit:development");
// No catalogue named: nothing is planned, because no bed will read one either.
assert.deepEqual(plannedRuntimes(["test/integration/a.test.ts"], {}, lab.root, answers), []);
} finally { lab.done(); }
});