import { readFileSync } from "node:fs"; // The module graph (novox/hq ADR 0070, ADR 0072). // // **This graph links module-versions to each other and knows nothing about nodes.** Which machine // runs what is the control plane's graph, and the two meet only when somebody installs something. // Keeping them apart is what lets the control plane carry on composing declarations while this is // down: it holds what it needs, and asks nothing here. // // A module-version is identified by its commit, not by a version string. A version is what somebody // calls a release; a commit is what was actually built (ADR 0009). import pg from "pg"; // `pg` is CommonJS: its default export is the module object, so Pool is a property of it. This is // the form that resolves under Node's ESM loader — the same shape model-usage uses. const { Pool } = pg; type PgPool = InstanceType; /** One module-version, as the builder delivered it. */ export interface ModuleVersion { module: string; commit: string; repository: string; /** The module's directory inside that repository (novox/hq ADR 0069). Empty is the root. */ path: string; ref: string; /** The manifest with its artifacts pinned — the module as the mesh should hold it. */ manifest: unknown; } /** An artifact a module-version produced, as the builder published it. */ export interface Made { name: string; kind: string; /** How anything else names it — for an image, a registry reference pinned to a digest. */ reference: string; } /** * A module whose artifacts were built against something that is no longer current. * * **An edge is an artifact reference, not a module name.** The builder can see exactly what it * built on top of — a pinned image — and cannot see which module produced it: that is a fact about * the graph, and the graph is here. So the reference is what is stored, and resolving it to a * module-version is a join against what each version says it made. An edge to an artifact no * module here produced is kept rather than dropped; it resolves by itself the day that module is * registered, which is the ordinary case while a mesh is still being filled in. */ export interface Stale { module: string; commit: string; /** What moved underneath it, and to where. */ because: { module: string; builtAgainst: string; nowAt: string }[]; } const DDL = ` CREATE TABLE IF NOT EXISTS module_version ( module text NOT NULL, commit_sha text NOT NULL, repository text NOT NULL DEFAULT '', path text NOT NULL DEFAULT '', ref text NOT NULL DEFAULT '', manifest jsonb NOT NULL DEFAULT '{}'::jsonb, registered timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (module, commit_sha) ); -- What the catalogue considers this module to be now. Separate from module_version because a -- module has many versions and exactly one current one, and "current" is a decision rather than -- a consequence of being newest. CREATE TABLE IF NOT EXISTS module_current ( module text PRIMARY KEY, commit_sha text NOT NULL, moved timestamptz NOT NULL DEFAULT now() ); -- What a module-version published. This is what makes a build edge resolvable: an edge names an -- artifact, and this says which version put that artifact there. CREATE TABLE IF NOT EXISTS module_artifact ( module text NOT NULL, commit_sha text NOT NULL, name text NOT NULL, kind text NOT NULL DEFAULT '', reference text NOT NULL, PRIMARY KEY (module, commit_sha, name) ); CREATE INDEX IF NOT EXISTS module_artifact_reference ON module_artifact (reference); -- What a module-version declares. Separate from the build edges above and never mixed with them: -- a build edge is a fact about what was compiled, discovered by building it, and these are -- intentions the module states about itself (ADR 0072). They answer different questions and go -- stale for different reasons. CREATE TABLE IF NOT EXISTS module_requires ( module text NOT NULL, commit_sha text NOT NULL, requirement text NOT NULL, PRIMARY KEY (module, commit_sha, requirement) ); CREATE TABLE IF NOT EXISTS module_provides ( module text NOT NULL, commit_sha text NOT NULL, provision text NOT NULL, scope text NOT NULL DEFAULT '', PRIMARY KEY (module, commit_sha, provision) ); CREATE INDEX IF NOT EXISTS module_provides_provision ON module_provides (provision); CREATE INDEX IF NOT EXISTS module_requires_requirement ON module_requires (requirement); CREATE TABLE IF NOT EXISTS built_against ( module text NOT NULL, commit_sha text NOT NULL, against_reference text NOT NULL, PRIMARY KEY (module, commit_sha, against_reference) ); `; // The graph first keyed its edges on a module and a commit, which the builder does not know and // never sent — so every build that had been built against anything was rejected, and the only // versions that registered were the ones built against nothing. No edge was ever stored, so there // is nothing to carry across: the old columns are dropped and the new one added. const MIGRATE = ` ALTER TABLE built_against ADD COLUMN IF NOT EXISTS against_reference text; DELETE FROM built_against WHERE against_reference IS NULL; ALTER TABLE built_against DROP COLUMN IF EXISTS against_module; ALTER TABLE built_against DROP COLUMN IF EXISTS against_commit; ALTER TABLE built_against ALTER COLUMN against_reference SET NOT NULL; -- After the column exists, and not in the schema above: on a store that still has the old shape, -- the table is not created, so an index named in the same breath would be built on a column that -- is not there yet. CREATE INDEX IF NOT EXISTS built_against_target ON built_against (against_reference); `; export class Graph { private constructor(private readonly pool: PgPool) {} static fromEnv(env: NodeJS.ProcessEnv = process.env): Graph { // As a file first (novox/hq ADR 0086): the connection string carries the password, and the // mesh writes it where only this process reads it; the plain variable remains for a hand-run. const url = env["DATABASE_URL"] ?? readMaybe(env["DATABASE_URL_FILE"]); if (!url) { throw new Error( "no DATABASE_URL: the catalogue holds the module graph and cannot hold it in memory, " + "because a graph that disappears on restart is not a record of anything", ); } return new Graph(new Pool({ connectionString: url })); } async migrate(): Promise { await this.pool.query(DDL); await this.pool.query(MIGRATE); } /** * Take a module-version the builder produced, and place it in the graph. * * **Returns whether this is an upgrade**, which is not the same as whether a build happened. A * rebuild producing the commit already current changes nothing, and announcing it as an upgrade * would ripple outward forever through modules that did not change (ADR 0072). */ async register(version: ModuleVersion, made: Made[], against: string[]): Promise<{ upgraded: boolean; previous: string | null }> { const client = await this.pool.connect(); try { await client.query("BEGIN"); await client.query( `INSERT INTO module_version (module, commit_sha, repository, path, ref, manifest) VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (module, commit_sha) DO UPDATE SET repository = excluded.repository, path = excluded.path, ref = excluded.ref, manifest = excluded.manifest`, [version.module, version.commit, version.repository, version.path, version.ref, JSON.stringify(version.manifest ?? {})], ); // Both are replaced rather than added to: they describe this build, and a previous build of // the same commit that saw different artifacts was wrong about one of them. // Read from the manifest rather than taken as a separate argument: the manifest the builder // resolved IS the module's declaration, and a second copy passed alongside it could disagree // with it. Replaced wholesale, because a version declares exactly one set of these. const declared = (version.manifest ?? {}) as { requires?: unknown[]; provides?: unknown[]; }; await client.query(`DELETE FROM module_requires WHERE module = $1 AND commit_sha = $2`, [version.module, version.commit]); for (const r of declared.requires ?? []) { const name = typeof r === "string" ? r : String((r as { name?: unknown }).name ?? ""); if (!name) continue; await client.query( `INSERT INTO module_requires (module, commit_sha, requirement) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING`, [version.module, version.commit, name]); } await client.query(`DELETE FROM module_provides WHERE module = $1 AND commit_sha = $2`, [version.module, version.commit]); for (const pv of declared.provides ?? []) { const name = typeof pv === "string" ? pv : String((pv as { name?: unknown }).name ?? ""); const scope = typeof pv === "string" ? "" : String((pv as { scope?: unknown }).scope ?? ""); if (!name) continue; await client.query( `INSERT INTO module_provides (module, commit_sha, provision, scope) VALUES ($1,$2,$3,$4) ON CONFLICT (module, commit_sha, provision) DO UPDATE SET scope = excluded.scope`, [version.module, version.commit, name, scope]); } await client.query(`DELETE FROM module_artifact WHERE module = $1 AND commit_sha = $2`, [version.module, version.commit]); for (const a of made) { await client.query( `INSERT INTO module_artifact (module, commit_sha, name, kind, reference) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (module, commit_sha, name) DO UPDATE SET kind = excluded.kind, reference = excluded.reference`, [version.module, version.commit, a.name, a.kind, a.reference]); } await client.query(`DELETE FROM built_against WHERE module = $1 AND commit_sha = $2`, [version.module, version.commit]); for (const reference of against) { await client.query( `INSERT INTO built_against (module, commit_sha, against_reference) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING`, [version.module, version.commit, reference]); } const was = await client.query( `SELECT commit_sha FROM module_current WHERE module = $1`, [version.module]); const previous = (was.rows[0]?.commit_sha as string | undefined) ?? null; const upgraded = previous !== version.commit; if (upgraded) { await client.query( `INSERT INTO module_current (module, commit_sha) VALUES ($1,$2) ON CONFLICT (module) DO UPDATE SET commit_sha = excluded.commit_sha, moved = now()`, [version.module, version.commit]); } await client.query("COMMIT"); return { upgraded, previous }; } catch (err) { await this.pool.query("ROLLBACK").catch(() => {}); throw err; } finally { client.release(); } } /** Every module the catalogue holds, with the commit it considers current. */ async modules(): Promise<{ module: string; commit: string; repository: string; path: string }[]> { const { rows } = await this.pool.query( `SELECT c.module, c.commit_sha, v.repository, v.path FROM module_current c JOIN module_version v ON v.module = c.module AND v.commit_sha = c.commit_sha ORDER BY c.module`); return rows.map((r) => ({ module: r.module as string, commit: r.commit_sha as string, repository: r.repository as string, path: r.path as string, })); } /** One module-version in full, current unless a commit is named. */ async show(module: string, commit?: string): Promise { const { rows } = await this.pool.query( commit ? `SELECT * FROM module_version WHERE module = $1 AND commit_sha = $2` : `SELECT v.* FROM module_version v JOIN module_current c ON c.module = v.module AND c.commit_sha = v.commit_sha WHERE v.module = $1`, commit ? [module, commit] : [module]); const r = rows[0]; if (!r) return null; return { module: r.module as string, commit: r.commit_sha as string, repository: r.repository as string, path: r.path as string, ref: r.ref as string, manifest: r.manifest, }; } /** * What was built against this module — the modules a change to it reaches. * * Resolved through the artifacts each version published, because that is what an edge names. */ async dependents(module: string): Promise<{ module: string; commit: string; againstCommit: string }[]> { // Distinct, because one artifact may have been published by more than one version of the // module that made it — several commits producing a byte-identical image — and a dependent // would otherwise be listed once per such commit, as though it were several dependents. const { rows } = await this.pool.query( `SELECT DISTINCT b.module, b.commit_sha, c2.commit_sha AS against_commit FROM built_against b JOIN module_current c ON c.module = b.module AND c.commit_sha = b.commit_sha JOIN module_artifact a ON a.reference = b.against_reference JOIN module_current c2 ON c2.module = a.module WHERE a.module = $1 ORDER BY b.module`, [module]); return rows.map((r) => ({ module: r.module as string, commit: r.commit_sha as string, againstCommit: r.against_commit as string, })); } /** * What must be rebuilt, and why. * * **The rule is a condition, not an order** (ADR 0072): a module is stale when anything it was * built against is no longer current. Asking this repeatedly is what produces the right order — * a module whose own dependencies are still stale simply stays stale until they are not, so a * chain and a diamond need no special handling and nothing has to know the shape in advance. */ async stale(): Promise { // **Compared by artifact, not by commit.** A commit moving is not the same as the thing it // produces moving: a change to a comment in a build recipe is a new commit and a byte-identical // image, and calling everything built on it stale would mean rebuilding the whole mesh to // arrive back exactly where it started. What makes a module stale is that the artifact it was // built against is no longer one the current version publishes. // // The producing version is picked as the earliest that published this artifact, because when // several commits produce the identical image, the first one is where it actually came from. const { rows } = await this.pool.query( `SELECT b.module, b.commit_sha, p.module AS against_module, p.commit_sha AS against_commit, c2.commit_sha AS now_at FROM built_against b JOIN module_current c1 ON c1.module = b.module AND c1.commit_sha = b.commit_sha JOIN LATERAL ( SELECT a.module, a.commit_sha FROM module_artifact a JOIN module_version v ON v.module = a.module AND v.commit_sha = a.commit_sha WHERE a.reference = b.against_reference ORDER BY v.registered LIMIT 1 ) p ON true JOIN module_current c2 ON c2.module = p.module WHERE NOT EXISTS ( SELECT 1 FROM module_artifact now WHERE now.module = c2.module AND now.commit_sha = c2.commit_sha AND now.reference = b.against_reference) ORDER BY b.module`); const by = new Map(); for (const r of rows) { const key = `${r.module}@${r.commit_sha}`; const entry = by.get(key) ?? { module: r.module as string, commit: r.commit_sha as string, because: [] }; entry.because.push({ module: r.against_module as string, builtAgainst: r.against_commit as string, nowAt: r.now_at as string, }); by.set(key, entry); } return [...by.values()]; } /** * Modules that are stale and whose own dependencies are all current — the ones that can be built * now. **This is the whole of ordering.** Anything not in this set is waiting for something else, * and will appear here once that thing is registered. */ async buildable(): Promise { const stale = await this.stale(); const waiting = new Set(stale.map((s) => s.module)); return stale.filter((s) => !s.because.some((b) => waiting.has(b.module))); } /** * What provides a given provision, and what needs it. * * **Answered from the graph rather than by scanning manifests**, which is how it is answered * today — a sweep over every module's declaration, which only whoever holds every declaration * can do and which is wrong the moment one of them changes. */ async whoProvides(provision: string): Promise<{ provides: { module: string; commit: string; scope: string }[]; requires: { module: string; commit: string }[]; }> { const provides = await this.pool.query( `SELECT p.module, p.commit_sha, p.scope FROM module_provides p JOIN module_current c ON c.module = p.module AND c.commit_sha = p.commit_sha WHERE p.provision = $1 ORDER BY p.module`, [provision]); const requires = await this.pool.query( `SELECT r.module, r.commit_sha FROM module_requires r JOIN module_current c ON c.module = r.module AND c.commit_sha = r.commit_sha WHERE r.requirement = $1 ORDER BY r.module`, [provision]); return { provides: provides.rows.map((r) => ({ module: r.module as string, commit: r.commit_sha as string, scope: r.scope as string })), requires: requires.rows.map((r) => ({ module: r.module as string, commit: r.commit_sha as string })), }; } async close(): Promise { await this.pool.end(); } } /** The content of a file the environment names, its line ending gone — or undefined when it names none. */ function readMaybe(path: string | undefined): string | undefined { if (!path) return undefined; try { return readFileSync(path, "utf8").replace(/\r?\n$/, ""); } catch { return undefined; } }