From a516ee847bb1b5cdec0c48c248fff489507293f7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 1 Sep 2026 01:20:14 +0200 Subject: [PATCH] Adopt a third-party workload, and keep a mesh between runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The adoption.** Software nobody here wrote, taking its credentials the way such software does — from its environment — and needing two containers that reach each other by name. The first module that could not have been declared this morning: it needs the network shape and it needs a sealed value to reach a container's environment. Its password is accepted rather than generated, which is the whole shape of an adoption: a service that already exists keeps the credential it already has. Asserted properly — a wrong password is refused by the same database, so the passing case means something. **The warm scenario.** A mesh kept between runs and returned to, which turned twelve minutes of bootstrap into thirty seconds of restore. Off unless asked for: a run that is meant to mean something raises from nothing. Its guard fired for real during this work, unprompted — a mesh-host commit landed and it refused the stale base, naming both commits, rather than passing tests against yesterday's binary. That is 04-ISSUES/005's rule one level down. Three things the guard learned the hard way and now handles: a snapshot captures disk and not memory, so the host is restarted after a restore and asserted to have come back; the stocked image digests are worked out while raising and a restored instance never raises, so they are kept; and comparing only the repositories this run can see clears the ones it cannot, so both directions are compared. The one real bug behind five failed attempts was in mesh-host and it reported itself precisely: a network shape the language had and no host implemented. Everything else was scaffolding of mine. --- scenarios/two-nodes.yml | 4 + src/cli.ts | 35 ++++++ src/warm.ts | 210 ++++++++++++++++++++++++++++++++++ test/integration/mesh.test.ts | 188 ++++++++++++++++++++++++++++-- test/warm.test.ts | 61 ++++++++++ 5 files changed, 490 insertions(+), 8 deletions(-) create mode 100644 src/warm.ts create mode 100644 test/warm.test.ts diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index 42858e5..225a0a6 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -28,6 +28,10 @@ images: # what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved # the same way. - registry:2 + # A real third-party workload, for adopting one the way the conversion will. Its database is + # the substrate's postgres image rather than its own: what is under test is the mesh delivering + # a module, not which postgres it delivers. + - ghcr.io/umami-software/umami:postgresql-latest # And the builder, because it is a module the mesh assigns rather than a program somebody # starts by hand — which is the only way its credential can be one the mesh delivered. - mesh-builder:development diff --git a/src/cli.ts b/src/cli.ts index 660fcb0..4bc4bcb 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -30,6 +30,8 @@ const USAGE = `mesh-lab — raise a disposable mesh on one machine diagram [out.drawio] draw what a scenario asks for diagram --live [out.drawio] draw what is actually raised + warm the scenario kept between runs, and whether it still counts + warm cool destroy it and forget it suite [paths...] [--no-build] rebuild the artifacts, run the end-to-end tests, leave a receipt last-run whether the last run still counts; non-zero when it does not @@ -118,6 +120,39 @@ async function main(): Promise { return; } + // A base state many tests start from, rather than each raising its own mesh. + // + // **The speed is the lesser half.** Tests that share one long-lived mesh accumulate each + // other's state, and a test that reads what the previous one left is a test that passes for + // the wrong reason — which has already happened here once. Returning to a named state between + // tests makes each of them independent. + case "warm": { + const { remembered, ready, cool } = await import("./warm.ts"); + const what = rest[0] ?? "status"; + if (what === "cool") { + const gone = await cool(); + console.log(gone ? `destroyed ${gone}, and forgot it` : "nothing was being kept warm"); + return; + } + const held = remembered(); + if (!held) { + console.log("nothing is being kept warm."); + console.log(" a scenario is warmed by whatever brought it to a state worth keeping;"); + console.log(" the integration suite does it when MESH_LAB_WARM is set."); + return; + } + console.log(`${held.instanceId} — ${held.scenario}, warmed ${held.at}`); + for (const [name, commit] of Object.entries(held.against)) { + console.log(` ${name.padEnd(14)} ${commit}`); + } + const said = await ready(held.scenario); + console.log(said.use === "restore" + ? "\n usable: it can be returned to" + : `\n NOT usable: ${said.why}`); + if (said.use !== "restore") process.exitCode = 1; + return; + } + case "base": { // `base build` exists because a sealed scenario cannot install a container runtime, and // the runtime has to come from somewhere with a network (novox/hq ADR 0006). diff --git a/src/warm.ts b/src/warm.ts new file mode 100644 index 0000000..f8762d0 --- /dev/null +++ b/src/warm.ts @@ -0,0 +1,210 @@ +/** + * A scenario kept between runs, already brought to a state worth starting from. + * + * **Bootstrapping a mesh takes minutes and proves the same thing every time.** The tests worth + * iterating on are the ones after it — assigning a module, adopting a workload, watching something + * fail. A warm instance is raised once, brought to that state, snapshotted, and restored on every + * later run in seconds. + * + * **The danger is precisely the one 04-ISSUES/005 is about**, one level down: a mesh snapshotted + * against yesterday's binaries will pass today's tests and report green, and nothing about the + * result would say what it was actually run against. So a warm instance records the commits it was + * built from, and is refused — not silently rebuilt, refused — when they have moved. + * + * **Fresh stays the default.** This is for iterating. A run that is meant to mean something raises + * from nothing, because "it passes" must not quietly come to mean "it passes against a mesh + * somebody bootstrapped last week". + */ + +import { readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { homedir } from "node:os"; + +import { list, restore, snapshot, snapshots, destroy } from "./lifecycle/operate.ts"; +import type { Against } from "./lastrun.ts"; +import { whatWasTested } from "./lastrun.ts"; + +/** The state a warm instance is kept at. One label, because a second is a state nobody named. */ +export const label = "warm"; + +export interface Warm { + scenario: string; + instanceId: string; + /** + * The image references the scenario's registry serves, pinned by digest. + * + * Kept because they are worked out while raising and a restored instance never raises. Without + * them a warm run knows nothing about what it can pull, and every test naming an image fails + * for a reason that has nothing to do with what it was testing. + */ + images: string[]; + /** The commit each repository was at when this was brought to its state. */ + against: Against; + at: string; +} + +/** Where the record lives: XDG state, beside the run receipt, for the same reason. */ +export function recordPath(): string { + const state = process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"); + return join(state, "mesh-lab", "warm.json"); +} + +export function remember(warm: Warm): void { + const path = recordPath(); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, JSON.stringify(warm, null, 2) + "\n"); +} + +export function remembered(): Warm | null { + try { + return JSON.parse(readFileSync(recordPath(), "utf8")) as Warm; + } catch { + return null; + } +} + +export function forget(): void { + rmSync(recordPath(), { force: true }); +} + +export type Verdict = + | { use: "restore"; instanceId: string } + | { use: "raise"; why: string }; + +/** + * judge decides whether a remembered instance may be restored. + * + * **Every reason to refuse is a reason a test would otherwise pass while meaning nothing**, so + * each is named rather than collapsed into "not usable". + */ +export function judge( + warm: Warm | null, + scenario: string, + standing: string[], + hasSnapshot: boolean, + against: Against, +): Verdict { + if (!warm) return { use: "raise", why: "nothing is being kept warm" }; + if (warm.scenario !== scenario) { + return { use: "raise", why: `what is kept warm is ${warm.scenario}, and this is ${scenario}` }; + } + if (!standing.includes(warm.instanceId)) { + return { use: "raise", why: `${warm.instanceId} is no longer standing` }; + } + if (!hasSnapshot) { + return { use: "raise", why: `${warm.instanceId} has no ${label} snapshot to return to` }; + } + // The check that keeps this honest. A mesh built from code that has since moved would pass + // today's tests against yesterday's binaries, and say nothing about it. + // + // **Both directions, because comparing only what is in front of you clears what is not.** The + // first version walked the current repositories alone, so running without the environment that + // names where they are compared nothing and reported the mesh usable — a warm instance built + // from code that had since moved, cleared by a check that had looked at neither. That is + // 04-ISSUES/005's rule again: a record that says nothing about something is not a record that + // clears it. + for (const name of new Set([...Object.keys(warm.against), ...Object.keys(against)])) { + const then = warm.against[name]; + const now = against[name]; + if (then === now) continue; + if (!now) { + return { + use: "raise", + why: `${name} was at ${then} when this was warmed, and nothing says where it is now — ` + + `so nothing can say whether it moved`, + }; + } + return { + use: "raise", + why: `${name} was at ${then ?? "nothing recorded"} when this was warmed, ` + + `and is now at ${now}`, + }; + } + return { use: "restore", instanceId: warm.instanceId }; +} + +/** What is standing right now, by instance. */ +export async function standingNow(): Promise { + return (await list()).map((i) => i.instanceId); +} + +/** + * ready returns an instance already at its warm state, or says why one must be raised. + * + * It never raises: raising needs a scenario, images and a bootstrap, and all of that belongs to + * whoever is using this rather than here. + */ +export async function ready( + scenario: string, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const warm = remembered(); + const standing = await standingNow(); + const has = warm ? (await snapshots(warm.instanceId)).includes(label) : false; + return judge(warm, scenario, standing, has, whatWasTested(env)); +} + +/** returnTo puts a warm instance back to its state, and says how long it took. */ +export async function returnTo( + instanceId: string, + log: (message: string) => void = () => {}, +): Promise { + const { usableSeconds } = await restore(instanceId, label, 180, log); + return usableSeconds; +} + +/** + * keep snapshots an instance as the state to come back to, and records what it was built from. + * + * Called once the caller has brought the scenario to whatever "ready to work" means for it. + */ +export async function keep( + scenario: string, + instanceId: string, + env: NodeJS.ProcessEnv = process.env, +): Promise { + await snapshot(instanceId, label); + const warm: Warm = { + scenario, + instanceId, + images: stockOf(instanceId), + against: whatWasTested(env), + at: new Date().toISOString(), + }; + remember(warm); + return warm; +} + +/** cool destroys what is being kept and forgets it. */ +export async function cool(): Promise { + const warm = remembered(); + forget(); + if (!warm) return null; + if ((await standingNow()).includes(warm.instanceId)) { + await destroy(warm.instanceId); + } + return warm.instanceId; +} + +/** + * What a raised scenario stocked, held until it is kept. + * + * Raising works the images out and snapshotting happens later, so this carries them between the + * two without the caller having to hold them. + */ +const stock = new Map(); + +export function rememberStock(instanceId: string, images: string[]): void { + stock.set(instanceId, images); +} + +function stockOf(instanceId: string): string[] { + return stock.get(instanceId) ?? []; +} + +/** What a restored instance's registry serves, from when it was warmed. */ +export function warmStock(instanceId: string): { images: string[] } { + const warm = remembered(); + if (!warm || warm.instanceId !== instanceId) return { images: [] }; + return { images: warm.images }; +} diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 4cc9e1d..057e6be 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -26,6 +26,10 @@ import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll } from "./harness.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; +import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts"; + +/** Whether this run keeps its mesh for the next one. Off unless asked for. */ +const warming = process.env["MESH_LAB_WARM"] === "1"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -128,6 +132,44 @@ function tokenFrom(said: string): string { before(async () => { if (skip) return; + + // A mesh kept between runs, when one is being kept and still counts. + // + // **Bootstrapping proves the same thing every time**, and the tests worth iterating on are the + // ones after it. Off by default: a run that is meant to mean something raises from nothing, + // because "it passes" must not come to mean "it passes against a mesh somebody bootstrapped + // last week". + if (warming) { + const said = await ready(SCENARIO); + if (said.use === "restore") { + instanceId = said.instanceId; + const seconds = await returnTo(instanceId); + stocked = warmStock(instanceId).images; + + // **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything + // this suite started by hand is gone — the host most of all. Without it the mesh looks + // perfectly healthy from the control plane's side: a module is assigned, a declaration is + // sent and recorded, and nothing on the machine is listening to apply it. That is exactly + // how this was first met, and it cost an hour to see. + // + // The real answer is a host started by init, which is what the design says it is anyway + // (novox/hq 05-the-node-host: a root service, installed as a package). Until the lab places + // it that way, the warm path restarts what it knows it started. + for (const machine of ["anchor", "laptop"]) { + await must(machine, `pgrep -x mesh-host >/dev/null || ` + + `(nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); + } + const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`); + assert.equal(running.out.trim(), "yes", + "the host did not come back after a restore, so nothing would apply anything"); + + console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` + + `and started the host again`); + return; + } + console.log(`warm: raising fresh — ${said.why}`); + } + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {}); instanceId = raised.instanceId; @@ -153,9 +195,20 @@ before(async () => { `MESH_WORKSPACE=/var/lib/mesh-builder ` + `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`); } + if (warming) { + // Snapshotted only now, with everything up: a state worth returning to is the one after the + // part nobody wants to repeat. + await rememberStock(instanceId, stocked); + const warm = await keep(SCENARIO, instanceId); + console.log(`warm: ${warm.instanceId} kept, against ` + + Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", ")); + } }, { timeout: 1_800_000 }); after(async () => { + // A kept instance survives on purpose, and `mesh-lab warm cool` is how it goes away. Everything + // else is destroyed, because an instance nobody meant to keep is one nobody will remember. + if (warming) return; if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); @@ -774,16 +827,16 @@ test("rotating a credential moves both ends, and the old one stops working", { // holding a matching string proves they agree; only an authentication proves they are right. const store = "/var/lib/mesh/postgres"; await must("anchor", `printf %s '{"module":"realstore","version":"1",` + - `"provides":[{"name":"realpostgres-database","scope":"mesh"}],` + + `"provides":[{"name":"real-postgres-database","scope":"mesh"}],` + `"capabilities":["container-runtime"],` + - `"serves":{"realpostgres-database":{"port":5433}},` + + `"serves":{"real-postgres-database":{"port":5433}},` + `"own-secrets":{"superuser":"${store}/superuser"},` + - `"grants":{"realpostgres-database":"${store}/grants"},` + + `"grants":{"real-postgres-database":"${store}/grants"},` + // Both halves. `grants` is where each consumer's sealed password lands; `receives` is the // manifest saying who asked and for what. Without the second the provisioner finds a // directory of unexplained secrets and says nothing has been granted — which is true, and // reads exactly like a credential that was never delivered. - `"receives":{"realpostgres-database":"${store}/grants/mesh.json"},` + + `"receives":{"real-postgres-database":"${store}/grants/mesh.json"},` + `"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` + `"resources":[` + `{"id":"state","type":"directory","path":"${store}","mode":"0755"},` + @@ -801,9 +854,9 @@ test("rotating a credential moves both ends, and the old one stops working", { `"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` + `> /tmp/realstore.json`); await must("anchor", `printf %s '{"module":"realapp","version":"1",` + - `"requires":["realpostgres-database"],"contributes":{"realpostgres-database":{"name":"realapp"}},` + - `"binds":{"realpostgres-database":"/etc/realapp/where.json"},` + - `"secrets":{"realpostgres-database":"/etc/realapp/password"},` + + `"requires":["real-postgres-database"],"contributes":{"real-postgres-database":{"name":"realapp"}},` + + `"binds":{"real-postgres-database":"/etc/realapp/where.json"},` + + `"secrets":{"real-postgres-database":"/etc/realapp/password"},` + `"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` + `> /tmp/realapp.json`); for (const f of ["realstore", "realapp"]) { @@ -856,7 +909,7 @@ test("rotating a credential moves both ends, and the old one stops working", { // Now rotate. One command: the record changes AND both ends are sent, because leaving the // sending to a later command is the fault above, exactly. - const said = await mesh("rotate realdatabase", 180_000); + const said = await mesh("rotate real-postgres-database", 180_000); assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`); assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`); await new Promise((r) => setTimeout(r, 25_000)); @@ -1486,3 +1539,122 @@ test("a service is reached by a name under the machine it runs on", { } await mesh("push"); }); + +// A real third-party workload, adopted the way the conversion will adopt one. +// +// **Everything before this used modules written to exercise the mesh.** This one is software +// nobody here wrote, taking its credentials the way such software does — from its environment — +// and needing two containers that reach each other by name. It is the first module that could not +// have been declared before today: it needs the `network` shape, and it needs a sealed value to +// reach a container's environment. +// +// Its database password is **accepted rather than generated**, which is the whole shape of an +// adoption: a service that already exists keeps the credential it already has, because minting a +// new one is how a running application stops being able to reach its own database. +test("a third-party workload is adopted, with the credential it already had", { + skip, timeout: 900_000, +}, async () => { + const password = "the-password-it-already-had"; + + await must("anchor", `printf %s ${quote(JSON.stringify({ + module: "umami", + version: "1", + capabilities: ["container-runtime"], + "own-secrets": { + database: "/var/lib/umami/database.env", + app: "/var/lib/umami/app.env", + }, + listens: [{ port: 1212, protocol: "tcp", from: "mesh", why: "the analytics page" }], + resources: [ + { id: "state", type: "directory", path: "/var/lib/umami", mode: "0700" }, + // The two containers must reach each other by name, which is what this shape is for. + { id: "net", type: "network", name: "umami" }, + { + id: "db", type: "container", name: "umami-db", + image: pinned("postgres"), + network: "umami", + env: { POSTGRES_DB: "umami", POSTGRES_USER: "umami" }, + "env-file": ["/var/lib/umami/database.env"], + }, + { + id: "app", type: "container", name: "umami", + image: pinned("ghcr.io/umami-software/umami"), + network: "umami", + env: { DATABASE_TYPE: "postgresql" }, + "env-file": ["/var/lib/umami/app.env"], + ports: ["1212:3000"], + }, + ], + }))} > /umami.json`); + await must("anchor", `docker cp /umami.json mesh-control:/umami.json`); + await mesh("module add /umami.json"); + + // **Accepted, not generated.** The value is what the database already answers to; the mesh + // seals it and cannot read it again. Given whole, as the environment lines the containers read. + await must("anchor", + `printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` + + `docker exec -i mesh-control /mesh-control secret accept anchor umami database --from -`); + await must("anchor", + `printf %s ${quote( + `DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` + + `docker exec -i mesh-control /mesh-control secret accept anchor umami app --from -`); + + await mesh("assign anchor umami"); + await mesh("push anchor", 300_000); + + // Both containers, and the network they share. + let up = false; + for (let i = 0; i < 60 && !up; i++) { + const running = await on("anchor", `docker ps --format '{{.Names}}'`); + up = running.out.includes("umami-db") && running.out.includes("umami"); + if (!up) await new Promise((r) => setTimeout(r, 5000)); + } + if (!up) { + // Everything that could say why, gathered before asserting. "It did not start" is the one + // thing already known; what is wanted is whether the mesh sent it, whether the host refused + // it, and what the runtime said when it tried. + const said = await mesh("status"); + const containers = await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`); + const applied = await on("anchor", + `${HOST_PATH} owned 2>&1 | head -30 || echo "the host could not say what it owns"`); + const files = await on("anchor", `ls -la /var/lib/umami/ 2>&1; ` + + `for f in /var/lib/umami/*.env; do echo "-- $f"; wc -c "$f"; done 2>&1`); + const tried = await on("anchor", + `docker inspect umami-db --format '{{.State.Status}} {{.State.Error}}' 2>&1; ` + + `docker logs umami-db 2>&1 | tail -15`); + assert.fail( + `the workload never started.\n\n` + + `── what the mesh thinks:\n${said}\n` + + `── containers:\n${containers.out}\n` + + `── what the host owns:\n${applied.out}\n` + + `── what the mesh wrote:\n${files.out}\n` + + `── the database container:\n${tried.out}\n`); + } + + // The environment file the mesh sealed is on the machine and readable only by root. + const mode = await must("anchor", `stat -c %a /var/lib/umami/database.env`); + assert.equal(mode.trim(), "600", "a file holding a credential is readable by more than root"); + + // **The assertion that matters: the credential works.** Not that a file arrived — that the + // database the mesh started answers to the password the mesh was given rather than one it made. + let connected = { out: "", ok: false }; + for (let i = 0; i < 40 && !connected.ok; i++) { + connected = await on("anchor", + `docker exec umami-db psql -U umami -d umami -qAt -c 'select 1'`); + if (!connected.ok) await new Promise((r) => setTimeout(r, 3000)); + } + assert.ok(connected.ok, `the database never came up:\n${connected.out}`); + + const wrong = await on("anchor", + `docker run --rm --network umami -e PGPASSWORD=not-the-password ${pinned("postgres")} ` + + `psql -h umami-db -U umami -d umami -qAt -c 'select 1'`); + assert.ok(!wrong.ok, + "the database accepted a password nobody gave it, so this proves nothing about the one that was"); + + // And the two containers reach each other by name over the module's own network. + const reached = await must("anchor", + `docker run --rm --network umami ${pinned("postgres")} ` + + `sh -c 'getent hosts umami-db || echo unreachable'`); + assert.doesNotMatch(reached, /unreachable/, + "a container could not reach the other by name, so the module's network did nothing"); +}); diff --git a/test/warm.test.ts b/test/warm.test.ts new file mode 100644 index 0000000..adf2207 --- /dev/null +++ b/test/warm.test.ts @@ -0,0 +1,61 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { judge, type Warm } from "../src/warm.ts"; + +const at = "2026-08-31T20:00:00Z"; +const built = { "mesh-lab": "aaa", "mesh-host": "bbb", "mesh-control": "ccc" }; +const warm = (over: Partial = {}): Warm => + ({ scenario: "two-nodes", instanceId: "mlab-two-nodes-1", images: [], against: built, at, ...over }); + +// The check this exists for: a mesh warmed against code that has since moved would pass today's +// tests against yesterday's binaries, and the result would say nothing about it. +// +// Same fault as novox/hq 04-ISSUES/005, one level down — a green result standing for a run +// against something other than what is in front of you. +test("a warm mesh built from code that has moved is refused", () => { + const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, + { ...built, "mesh-host": "moved" }); + assert.equal(said.use, "raise"); + assert.match(said.use === "raise" ? said.why : "", /mesh-host was at bbb.*now at moved/); +}); + +test("a warm mesh built from the same code is used", () => { + const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, built); + assert.equal(said.use, "restore"); +}); + +// Each refusal is named, because each is a different thing being wrong. +test("every reason to raise instead says which reason it was", () => { + const cases: [string, ReturnType][] = [ + ["nothing kept", judge(null, "two-nodes", [], true, built)], + ["another scenario", judge(warm({ scenario: "first-node" }), "two-nodes", + ["mlab-two-nodes-1"], true, built)], + ["not standing", judge(warm(), "two-nodes", [], true, built)], + ["no snapshot", judge(warm(), "two-nodes", ["mlab-two-nodes-1"], false, built)], + ]; + for (const [what, said] of cases) { + assert.equal(said.use, "raise", what); + assert.ok(said.use === "raise" && said.why.length > 10, + `${what} was refused without saying why: ${JSON.stringify(said)}`); + } +}); + +// A repository the warm record never accounted for is a difference, not a match. +test("a repository that was not recorded when it was warmed is refused", () => { + const said = judge(warm({ against: { "mesh-lab": "aaa" } }), "two-nodes", + ["mlab-two-nodes-1"], true, built); + assert.equal(said.use, "raise"); +}); + +// A repository the current run cannot see is not a repository that agrees. +// +// **Found by testing the guard rather than trusting it.** The first version walked only the +// repositories the current environment names, so running without that environment compared +// nothing and called a stale mesh usable. The mesh had genuinely moved; the check had looked at +// neither side. +test("a repository this run cannot locate is refused, not passed over", () => { + const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, { "mesh-lab": "aaa" }); + assert.equal(said.use, "raise"); + assert.match(said.use === "raise" ? said.why : "", + /nothing says where it is now|so nothing can say whether it moved/); +});