A provisioner removes nothing while its contributions file cannot be used

An unreadable, unparsable or unrecognised contributions file was read as "no consumer asks", and the
reconcile pass then removed every consumer it had applied. On 2026-10-04 one such read made the
postgres provider drop every database on the control node. Such a pass now applies and removes
nothing and says why, once; only a file read with a given list withdraws a consumer, and every
removal is logged before it is made.
This commit is contained in:
2026-10-05 01:28:57 +02:00
parent c253d76321
commit 2865d53991
2 changed files with 94 additions and 16 deletions
+43 -15
View File
@@ -120,9 +120,23 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
// like a wait.
const waiting = new Map<string, number>();
let stopped = false;
// A reason the contributions file could not be used, said once until it changes.
let lastWarning = "";
const warn = (why: string): void => {
if (why === lastWarning) return;
if (why) console.error(`[provisioner:${resource}] ${why}; nothing applied or removed until it can be read`);
else console.error(`[provisioner:${resource}] contributions file readable again`);
lastWarning = why;
};
async function reconcile(): Promise<void> {
const given = await readContributions(receives, resource);
const given = await readContributions(receives, resource, warn);
// **Nothing read is not nobody asking** (novox/hq issue 241). A file that could not be read,
// parsed or recognised says nothing about which consumers the mesh wants, and treating it as
// "none" withdrew every consumer at once: on 2026-10-04 one failed read made the postgres provider
// drop every database it held. Only a file that was read can withdraw anything, so the whole pass
// waits for the next one.
if (given === null) return;
const wantByAs = new Map(given.map((g) => [g.as, g]));
// On this pass, ask the backend rather than memory whether each applied consumer is still there.
let verifying = adapter.holds !== undefined && Date.now() - verifiedAt >= verifyEveryMs;
@@ -225,6 +239,8 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
// working for ever. Only logins this harness created are touched.
for (const [as, was] of [...applied.entries()]) {
if (wantByAs.has(as)) continue;
// Said before it is done: a removal is the one act here that destroys what a consumer kept.
console.error(`[provisioner:${resource}] ${as}: no longer in ${receives}; removing it from the backend`);
try {
await adapter.remove({ as, derived: was.derived });
applied.delete(as);
@@ -252,30 +268,42 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
}
/**
* Read the mesh's contributions file for one resource. Absent or empty means "no consumer asks for
* this" — the file is always written, so a provider can tell that from "the mesh never wrote it".
* Read the mesh's contributions file for one resource: the consumers it asks for, or `null` when the
* file says nothing usable — absent, unreadable, not JSON, for another requirement, or without a
* `given` list. `null` is not "no consumer": the mesh always writes the file, and writes `given: []`
* when nobody asks, so only that withdraws every consumer.
*/
async function readContributions(path: string, resource: string): Promise<Contribution[]> {
async function readContributions(
path: string,
resource: string,
warn: (why: string) => void,
): Promise<Contribution[] | null> {
let raw: string;
try {
raw = await readFile(path, "utf8");
} catch {
return []; // not written yet, or nobody serves anything here — nothing to converge to
}
let doc: { requirement?: string; given?: Contribution[] };
try {
doc = JSON.parse(raw) as { requirement?: string; given?: Contribution[] };
} catch (err) {
console.error(`[provisioner:${resource}] contributions file is not JSON (${path}): ${err}`);
return [];
warn(`contributions file unreadable (${path}): ${(err as Error).message}`);
return null;
}
let doc: { requirement?: string; given?: unknown };
try {
doc = JSON.parse(raw) as { requirement?: string; given?: unknown };
} catch (err) {
warn(`contributions file is not JSON (${path}): ${err}`);
return null;
}
if (doc.requirement && doc.requirement !== resource) {
console.error(`[provisioner:${resource}] ${path} is for ${doc.requirement}, not ${resource}`);
return [];
warn(`${path} is for ${doc.requirement}, not ${resource}`);
return null;
}
if (!Array.isArray(doc.given)) {
warn(`${path} has no given list`);
return null;
}
warn("");
// A contribution with no `as` is not a credential grant (a module offering something on its own
// machine) — the provisioner has nothing to create for it.
return (doc.given ?? []).filter((g) => g.as && g.secret);
return (doc.given as Contribution[]).filter((g) => g.as && g.secret);
}
/** A check that did not answer in time: the backend, not the consumer, is the likely cause. */
+51 -1
View File
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, writeFile, readFile, readdir } from "node:fs/promises";
import { mkdtemp, writeFile, readFile, readdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -160,6 +160,56 @@ test("provisioner stops calling an unreadable secret a race, and says what to lo
assert.ok(loud[0].includes("issue 225"), "the loud line does not name the record");
});
test("provisioner removes nothing while its contributions file cannot be used (novox/hq issue 241)", async () => {
const dir = await mkdtemp(join(tmpdir(), "prov-"));
const created: string[] = [];
const removed: string[] = [];
await writeFile(join(dir, "a.secret"), "pw-a\n");
await writeFile(join(dir, "b.secret"), "pw-b\n");
const receives = join(dir, "postgres-database.json");
const doc = (given: unknown[]): string =>
JSON.stringify({ contributions: 1, requirement: "postgres-database", given });
const both = doc([
{ from: "forge", node: "anchor", as: "mesh_anchor_forge", secret: join(dir, "a.secret") },
{ from: "mail", node: "anchor", as: "mesh_anchor_mail", secret: join(dir, "b.secret") },
]);
await writeFile(receives, both);
const stop = runProvisioner(
"postgres-database",
{
async create(p) {
created.push(p.as);
},
async remove(p) {
removed.push(p.as);
},
},
{ receives, everyMs: 20 },
);
await waitFor(() => created.length === 2, 2000);
// Each way the file can say nothing usable, held for many passes: nothing is withdrawn.
for (const broken of [
async () => rm(receives),
async () => writeFile(receives, "{ half a fi"),
async () => writeFile(receives, JSON.stringify({ contributions: 1, requirement: "s3-bucket", given: [] })),
async () => writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database" })),
]) {
await broken();
await new Promise((r) => setTimeout(r, 200));
assert.deepEqual(removed, [], "a consumer was removed while the file could not be used");
await writeFile(receives, both);
await new Promise((r) => setTimeout(r, 100));
}
// The file read and saying so still withdraws: an explicit empty list removes both.
await writeFile(receives, doc([]));
await waitFor(() => removed.length === 2, 2000);
assert.deepEqual([...removed].sort(), ["mesh_anchor_forge", "mesh_anchor_mail"]);
stop();
});
test("provisioner applies again what the backend no longer holds, and trusts memory without holds", async () => {
const dir = await mkdtemp(join(tmpdir(), "prov-holds-"));
await writeFile(join(dir, "webapp.secret"), "minted-pw\n");