jschoubben
1a28e5aec6
From the survey of every env-file secret (ADR 0086, issue 041): amqp-ping, minio, mongodb and grafana use the _FILE twin their software honours; mesh-catalog and model-usage read DATABASE_URL_FILE (a file the mesh templates, mounted where only the runtime reads it); grafana's secret files belong to its own account. Two dead deliveries removed: a line nothing read in amqp-email-forwarder, and mailu's secret.env on four containers that never read it. The 25 exceptions that remain carry the surveyed reason — convertible and awaiting a bed, convertible through a generated config file, the application's own code, or not convertible.
406 lines
18 KiB
TypeScript
406 lines
18 KiB
TypeScript
import { readFileSync } from "node:fs";
|
|
// The module graph (novox/hq ADR 0070, ADR 0072).
|
|
//
|
|
// **This graph links module-versions to each other and knows nothing about nodes.** Which machine
|
|
// runs what is the control plane's graph, and the two meet only when somebody installs something.
|
|
// Keeping them apart is what lets the control plane carry on composing declarations while this is
|
|
// down: it holds what it needs, and asks nothing here.
|
|
//
|
|
// A module-version is identified by its commit, not by a version string. A version is what somebody
|
|
// calls a release; a commit is what was actually built (ADR 0009).
|
|
|
|
import pg from "pg";
|
|
|
|
// `pg` is CommonJS: its default export is the module object, so Pool is a property of it. This is
|
|
// the form that resolves under Node's ESM loader — the same shape model-usage uses.
|
|
const { Pool } = pg;
|
|
type PgPool = InstanceType<typeof Pool>;
|
|
|
|
/** One module-version, as the builder delivered it. */
|
|
export interface ModuleVersion {
|
|
module: string;
|
|
commit: string;
|
|
repository: string;
|
|
/** The module's directory inside that repository (novox/hq ADR 0069). Empty is the root. */
|
|
path: string;
|
|
ref: string;
|
|
/** The manifest with its artifacts pinned — the module as the mesh should hold it. */
|
|
manifest: unknown;
|
|
}
|
|
|
|
/** An artifact a module-version produced, as the builder published it. */
|
|
export interface Made {
|
|
name: string;
|
|
kind: string;
|
|
/** How anything else names it — for an image, a registry reference pinned to a digest. */
|
|
reference: string;
|
|
}
|
|
|
|
/**
|
|
* A module whose artifacts were built against something that is no longer current.
|
|
*
|
|
* **An edge is an artifact reference, not a module name.** The builder can see exactly what it
|
|
* built on top of — a pinned image — and cannot see which module produced it: that is a fact about
|
|
* the graph, and the graph is here. So the reference is what is stored, and resolving it to a
|
|
* module-version is a join against what each version says it made. An edge to an artifact no
|
|
* module here produced is kept rather than dropped; it resolves by itself the day that module is
|
|
* registered, which is the ordinary case while a mesh is still being filled in.
|
|
*/
|
|
|
|
export interface Stale {
|
|
module: string;
|
|
commit: string;
|
|
/** What moved underneath it, and to where. */
|
|
because: { module: string; builtAgainst: string; nowAt: string }[];
|
|
}
|
|
|
|
const DDL = `
|
|
CREATE TABLE IF NOT EXISTS module_version (
|
|
module text NOT NULL,
|
|
commit_sha text NOT NULL,
|
|
repository text NOT NULL DEFAULT '',
|
|
path text NOT NULL DEFAULT '',
|
|
ref text NOT NULL DEFAULT '',
|
|
manifest jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
registered timestamptz NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (module, commit_sha)
|
|
);
|
|
|
|
-- What the catalogue considers this module to be now. Separate from module_version because a
|
|
-- module has many versions and exactly one current one, and "current" is a decision rather than
|
|
-- a consequence of being newest.
|
|
CREATE TABLE IF NOT EXISTS module_current (
|
|
module text PRIMARY KEY,
|
|
commit_sha text NOT NULL,
|
|
moved timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
|
|
-- What a module-version published. This is what makes a build edge resolvable: an edge names an
|
|
-- artifact, and this says which version put that artifact there.
|
|
CREATE TABLE IF NOT EXISTS module_artifact (
|
|
module text NOT NULL,
|
|
commit_sha text NOT NULL,
|
|
name text NOT NULL,
|
|
kind text NOT NULL DEFAULT '',
|
|
reference text NOT NULL,
|
|
PRIMARY KEY (module, commit_sha, name)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS module_artifact_reference ON module_artifact (reference);
|
|
|
|
-- What a module-version declares. Separate from the build edges above and never mixed with them:
|
|
-- a build edge is a fact about what was compiled, discovered by building it, and these are
|
|
-- intentions the module states about itself (ADR 0072). They answer different questions and go
|
|
-- stale for different reasons.
|
|
CREATE TABLE IF NOT EXISTS module_requires (
|
|
module text NOT NULL,
|
|
commit_sha text NOT NULL,
|
|
requirement text NOT NULL,
|
|
PRIMARY KEY (module, commit_sha, requirement)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS module_provides (
|
|
module text NOT NULL,
|
|
commit_sha text NOT NULL,
|
|
provision text NOT NULL,
|
|
scope text NOT NULL DEFAULT '',
|
|
PRIMARY KEY (module, commit_sha, provision)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS module_provides_provision ON module_provides (provision);
|
|
CREATE INDEX IF NOT EXISTS module_requires_requirement ON module_requires (requirement);
|
|
|
|
CREATE TABLE IF NOT EXISTS built_against (
|
|
module text NOT NULL,
|
|
commit_sha text NOT NULL,
|
|
against_reference text NOT NULL,
|
|
PRIMARY KEY (module, commit_sha, against_reference)
|
|
);
|
|
`;
|
|
|
|
// The graph first keyed its edges on a module and a commit, which the builder does not know and
|
|
// never sent — so every build that had been built against anything was rejected, and the only
|
|
// versions that registered were the ones built against nothing. No edge was ever stored, so there
|
|
// is nothing to carry across: the old columns are dropped and the new one added.
|
|
const MIGRATE = `
|
|
ALTER TABLE built_against ADD COLUMN IF NOT EXISTS against_reference text;
|
|
DELETE FROM built_against WHERE against_reference IS NULL;
|
|
ALTER TABLE built_against DROP COLUMN IF EXISTS against_module;
|
|
ALTER TABLE built_against DROP COLUMN IF EXISTS against_commit;
|
|
ALTER TABLE built_against ALTER COLUMN against_reference SET NOT NULL;
|
|
|
|
-- After the column exists, and not in the schema above: on a store that still has the old shape,
|
|
-- the table is not created, so an index named in the same breath would be built on a column that
|
|
-- is not there yet.
|
|
CREATE INDEX IF NOT EXISTS built_against_target ON built_against (against_reference);
|
|
`;
|
|
|
|
export class Graph {
|
|
private constructor(private readonly pool: PgPool) {}
|
|
|
|
static fromEnv(env: NodeJS.ProcessEnv = process.env): Graph {
|
|
// As a file first (novox/hq ADR 0086): the connection string carries the password, and the
|
|
// mesh writes it where only this process reads it; the plain variable remains for a hand-run.
|
|
const url = env["DATABASE_URL"] ?? readMaybe(env["DATABASE_URL_FILE"]);
|
|
if (!url) {
|
|
throw new Error(
|
|
"no DATABASE_URL: the catalogue holds the module graph and cannot hold it in memory, " +
|
|
"because a graph that disappears on restart is not a record of anything",
|
|
);
|
|
}
|
|
return new Graph(new Pool({ connectionString: url }));
|
|
}
|
|
|
|
async migrate(): Promise<void> {
|
|
await this.pool.query(DDL);
|
|
await this.pool.query(MIGRATE);
|
|
}
|
|
|
|
/**
|
|
* Take a module-version the builder produced, and place it in the graph.
|
|
*
|
|
* **Returns whether this is an upgrade**, which is not the same as whether a build happened. A
|
|
* rebuild producing the commit already current changes nothing, and announcing it as an upgrade
|
|
* would ripple outward forever through modules that did not change (ADR 0072).
|
|
*/
|
|
async register(version: ModuleVersion, made: Made[], against: string[]): Promise<{ upgraded: boolean; previous: string | null }> {
|
|
const client = await this.pool.connect();
|
|
try {
|
|
await client.query("BEGIN");
|
|
await client.query(
|
|
`INSERT INTO module_version (module, commit_sha, repository, path, ref, manifest)
|
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
ON CONFLICT (module, commit_sha) DO UPDATE SET
|
|
repository = excluded.repository, path = excluded.path,
|
|
ref = excluded.ref, manifest = excluded.manifest`,
|
|
[version.module, version.commit, version.repository, version.path, version.ref,
|
|
JSON.stringify(version.manifest ?? {})],
|
|
);
|
|
|
|
// Both are replaced rather than added to: they describe this build, and a previous build of
|
|
// the same commit that saw different artifacts was wrong about one of them.
|
|
// Read from the manifest rather than taken as a separate argument: the manifest the builder
|
|
// resolved IS the module's declaration, and a second copy passed alongside it could disagree
|
|
// with it. Replaced wholesale, because a version declares exactly one set of these.
|
|
const declared = (version.manifest ?? {}) as {
|
|
requires?: unknown[];
|
|
provides?: unknown[];
|
|
};
|
|
await client.query(`DELETE FROM module_requires WHERE module = $1 AND commit_sha = $2`,
|
|
[version.module, version.commit]);
|
|
for (const r of declared.requires ?? []) {
|
|
const name = typeof r === "string" ? r : String((r as { name?: unknown }).name ?? "");
|
|
if (!name) continue;
|
|
await client.query(
|
|
`INSERT INTO module_requires (module, commit_sha, requirement) VALUES ($1,$2,$3)
|
|
ON CONFLICT DO NOTHING`, [version.module, version.commit, name]);
|
|
}
|
|
await client.query(`DELETE FROM module_provides WHERE module = $1 AND commit_sha = $2`,
|
|
[version.module, version.commit]);
|
|
for (const pv of declared.provides ?? []) {
|
|
const name = typeof pv === "string" ? pv : String((pv as { name?: unknown }).name ?? "");
|
|
const scope = typeof pv === "string" ? "" : String((pv as { scope?: unknown }).scope ?? "");
|
|
if (!name) continue;
|
|
await client.query(
|
|
`INSERT INTO module_provides (module, commit_sha, provision, scope) VALUES ($1,$2,$3,$4)
|
|
ON CONFLICT (module, commit_sha, provision) DO UPDATE SET scope = excluded.scope`,
|
|
[version.module, version.commit, name, scope]);
|
|
}
|
|
|
|
await client.query(`DELETE FROM module_artifact WHERE module = $1 AND commit_sha = $2`,
|
|
[version.module, version.commit]);
|
|
for (const a of made) {
|
|
await client.query(
|
|
`INSERT INTO module_artifact (module, commit_sha, name, kind, reference)
|
|
VALUES ($1,$2,$3,$4,$5) ON CONFLICT (module, commit_sha, name) DO UPDATE SET
|
|
kind = excluded.kind, reference = excluded.reference`,
|
|
[version.module, version.commit, a.name, a.kind, a.reference]);
|
|
}
|
|
|
|
await client.query(`DELETE FROM built_against WHERE module = $1 AND commit_sha = $2`,
|
|
[version.module, version.commit]);
|
|
for (const reference of against) {
|
|
await client.query(
|
|
`INSERT INTO built_against (module, commit_sha, against_reference)
|
|
VALUES ($1,$2,$3) ON CONFLICT DO NOTHING`,
|
|
[version.module, version.commit, reference]);
|
|
}
|
|
|
|
const was = await client.query(
|
|
`SELECT commit_sha FROM module_current WHERE module = $1`, [version.module]);
|
|
const previous = (was.rows[0]?.commit_sha as string | undefined) ?? null;
|
|
const upgraded = previous !== version.commit;
|
|
if (upgraded) {
|
|
await client.query(
|
|
`INSERT INTO module_current (module, commit_sha) VALUES ($1,$2)
|
|
ON CONFLICT (module) DO UPDATE SET commit_sha = excluded.commit_sha, moved = now()`,
|
|
[version.module, version.commit]);
|
|
}
|
|
await client.query("COMMIT");
|
|
return { upgraded, previous };
|
|
} catch (err) {
|
|
await this.pool.query("ROLLBACK").catch(() => {});
|
|
throw err;
|
|
} finally {
|
|
client.release();
|
|
}
|
|
}
|
|
|
|
/** Every module the catalogue holds, with the commit it considers current. */
|
|
async modules(): Promise<{ module: string; commit: string; repository: string; path: string }[]> {
|
|
const { rows } = await this.pool.query(
|
|
`SELECT c.module, c.commit_sha, v.repository, v.path
|
|
FROM module_current c
|
|
JOIN module_version v ON v.module = c.module AND v.commit_sha = c.commit_sha
|
|
ORDER BY c.module`);
|
|
return rows.map((r) => ({
|
|
module: r.module as string, commit: r.commit_sha as string,
|
|
repository: r.repository as string, path: r.path as string,
|
|
}));
|
|
}
|
|
|
|
/** One module-version in full, current unless a commit is named. */
|
|
async show(module: string, commit?: string): Promise<ModuleVersion | null> {
|
|
const { rows } = await this.pool.query(
|
|
commit
|
|
? `SELECT * FROM module_version WHERE module = $1 AND commit_sha = $2`
|
|
: `SELECT v.* FROM module_version v JOIN module_current c
|
|
ON c.module = v.module AND c.commit_sha = v.commit_sha WHERE v.module = $1`,
|
|
commit ? [module, commit] : [module]);
|
|
const r = rows[0];
|
|
if (!r) return null;
|
|
return {
|
|
module: r.module as string, commit: r.commit_sha as string,
|
|
repository: r.repository as string, path: r.path as string,
|
|
ref: r.ref as string, manifest: r.manifest,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* What was built against this module — the modules a change to it reaches.
|
|
*
|
|
* Resolved through the artifacts each version published, because that is what an edge names.
|
|
*/
|
|
async dependents(module: string): Promise<{ module: string; commit: string; againstCommit: string }[]> {
|
|
// Distinct, because one artifact may have been published by more than one version of the
|
|
// module that made it — several commits producing a byte-identical image — and a dependent
|
|
// would otherwise be listed once per such commit, as though it were several dependents.
|
|
const { rows } = await this.pool.query(
|
|
`SELECT DISTINCT b.module, b.commit_sha, c2.commit_sha AS against_commit
|
|
FROM built_against b
|
|
JOIN module_current c ON c.module = b.module AND c.commit_sha = b.commit_sha
|
|
JOIN module_artifact a ON a.reference = b.against_reference
|
|
JOIN module_current c2 ON c2.module = a.module
|
|
WHERE a.module = $1
|
|
ORDER BY b.module`, [module]);
|
|
return rows.map((r) => ({
|
|
module: r.module as string, commit: r.commit_sha as string,
|
|
againstCommit: r.against_commit as string,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* What must be rebuilt, and why.
|
|
*
|
|
* **The rule is a condition, not an order** (ADR 0072): a module is stale when anything it was
|
|
* built against is no longer current. Asking this repeatedly is what produces the right order —
|
|
* a module whose own dependencies are still stale simply stays stale until they are not, so a
|
|
* chain and a diamond need no special handling and nothing has to know the shape in advance.
|
|
*/
|
|
async stale(): Promise<Stale[]> {
|
|
// **Compared by artifact, not by commit.** A commit moving is not the same as the thing it
|
|
// produces moving: a change to a comment in a build recipe is a new commit and a byte-identical
|
|
// image, and calling everything built on it stale would mean rebuilding the whole mesh to
|
|
// arrive back exactly where it started. What makes a module stale is that the artifact it was
|
|
// built against is no longer one the current version publishes.
|
|
//
|
|
// The producing version is picked as the earliest that published this artifact, because when
|
|
// several commits produce the identical image, the first one is where it actually came from.
|
|
const { rows } = await this.pool.query(
|
|
`SELECT b.module, b.commit_sha, p.module AS against_module, p.commit_sha AS against_commit,
|
|
c2.commit_sha AS now_at
|
|
FROM built_against b
|
|
JOIN module_current c1 ON c1.module = b.module AND c1.commit_sha = b.commit_sha
|
|
JOIN LATERAL (
|
|
SELECT a.module, a.commit_sha
|
|
FROM module_artifact a
|
|
JOIN module_version v ON v.module = a.module AND v.commit_sha = a.commit_sha
|
|
WHERE a.reference = b.against_reference
|
|
ORDER BY v.registered
|
|
LIMIT 1
|
|
) p ON true
|
|
JOIN module_current c2 ON c2.module = p.module
|
|
WHERE NOT EXISTS (
|
|
SELECT 1 FROM module_artifact now
|
|
WHERE now.module = c2.module AND now.commit_sha = c2.commit_sha
|
|
AND now.reference = b.against_reference)
|
|
ORDER BY b.module`);
|
|
const by = new Map<string, Stale>();
|
|
for (const r of rows) {
|
|
const key = `${r.module}@${r.commit_sha}`;
|
|
const entry = by.get(key) ?? { module: r.module as string, commit: r.commit_sha as string, because: [] };
|
|
entry.because.push({
|
|
module: r.against_module as string,
|
|
builtAgainst: r.against_commit as string,
|
|
nowAt: r.now_at as string,
|
|
});
|
|
by.set(key, entry);
|
|
}
|
|
return [...by.values()];
|
|
}
|
|
|
|
/**
|
|
* Modules that are stale and whose own dependencies are all current — the ones that can be built
|
|
* now. **This is the whole of ordering.** Anything not in this set is waiting for something else,
|
|
* and will appear here once that thing is registered.
|
|
*/
|
|
async buildable(): Promise<Stale[]> {
|
|
const stale = await this.stale();
|
|
const waiting = new Set(stale.map((s) => s.module));
|
|
return stale.filter((s) => !s.because.some((b) => waiting.has(b.module)));
|
|
}
|
|
|
|
/**
|
|
* What provides a given provision, and what needs it.
|
|
*
|
|
* **Answered from the graph rather than by scanning manifests**, which is how it is answered
|
|
* today — a sweep over every module's declaration, which only whoever holds every declaration
|
|
* can do and which is wrong the moment one of them changes.
|
|
*/
|
|
async whoProvides(provision: string): Promise<{
|
|
provides: { module: string; commit: string; scope: string }[];
|
|
requires: { module: string; commit: string }[];
|
|
}> {
|
|
const provides = await this.pool.query(
|
|
`SELECT p.module, p.commit_sha, p.scope
|
|
FROM module_provides p
|
|
JOIN module_current c ON c.module = p.module AND c.commit_sha = p.commit_sha
|
|
WHERE p.provision = $1 ORDER BY p.module`, [provision]);
|
|
const requires = await this.pool.query(
|
|
`SELECT r.module, r.commit_sha
|
|
FROM module_requires r
|
|
JOIN module_current c ON c.module = r.module AND c.commit_sha = r.commit_sha
|
|
WHERE r.requirement = $1 ORDER BY r.module`, [provision]);
|
|
return {
|
|
provides: provides.rows.map((r) => ({
|
|
module: r.module as string, commit: r.commit_sha as string, scope: r.scope as string })),
|
|
requires: requires.rows.map((r) => ({
|
|
module: r.module as string, commit: r.commit_sha as string })),
|
|
};
|
|
}
|
|
|
|
async close(): Promise<void> {
|
|
await this.pool.end();
|
|
}
|
|
}
|
|
|
|
/** The content of a file the environment names, its line ending gone — or undefined when it names none. */
|
|
function readMaybe(path: string | undefined): string | undefined {
|
|
if (!path) return undefined;
|
|
try {
|
|
return readFileSync(path, "utf8").replace(/\r?\n$/, "");
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|