From a73cb8a2f82ff55c7a119a12518eae63343ec707 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 12 Sep 2026 23:16:05 +0200 Subject: [PATCH] The catalogue, as a module that owns the module graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It links module-versions to each other and knows nothing about nodes; which machine runs what stays the control plane's (novox/hq ADR 0070, 0072). Keeping them apart is what lets the control plane carry on composing declarations while this is down. The builder announces what it built, this places it in the graph and announces what that means, and the control plane hooks the meaning rather than the build output. A rebuild producing the commit already current is registered and is not an upgrade — announcing it would ripple outward forever through modules that did not change. Ordering is not computed. Modules stale and waiting on nothing that is itself stale are announced as buildable; the rest stay stale and appear once whatever they were waiting for is registered, so a chain and a diamond need no special handling and nothing holds a plan. Four tools over the graph: what this mesh holds, one module in full, what a change to a module reaches, and what must be rebuilt and why. The edges are derived from builds rather than declared, so they cannot drift from what the code actually uses. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- modules/mesh-catalog/index.ts | 79 ++++++++++ modules/mesh-catalog/module.json | 77 +++++++++ modules/mesh-catalog/package.json | 15 ++ modules/mesh-catalog/pg.d.ts | 22 +++ modules/mesh-catalog/store.ts | 236 ++++++++++++++++++++++++++++ modules/mesh-catalog/tools/index.ts | 67 ++++++++ modules/mesh-catalog/tsconfig.json | 17 ++ 7 files changed, 513 insertions(+) create mode 100644 modules/mesh-catalog/index.ts create mode 100644 modules/mesh-catalog/module.json create mode 100644 modules/mesh-catalog/package.json create mode 100644 modules/mesh-catalog/pg.d.ts create mode 100644 modules/mesh-catalog/store.ts create mode 100644 modules/mesh-catalog/tools/index.ts create mode 100644 modules/mesh-catalog/tsconfig.json diff --git a/modules/mesh-catalog/index.ts b/modules/mesh-catalog/index.ts new file mode 100644 index 0000000..7d7f29d --- /dev/null +++ b/modules/mesh-catalog/index.ts @@ -0,0 +1,79 @@ +// mesh-catalog's entrypoint — the module graph's consumer (novox/hq ADR 0070, ADR 0072). +// +// The builder announces what it built; this places it in the graph and announces what that means. +// The control plane hooks the *meaning* — a module was upgraded — rather than the build output, so +// it never has to interpret an artifact or ask this module anything. +// +// **Ordering is not computed here.** When a registration makes something else stale, the modules +// whose own dependencies are all current are announced as needing a rebuild; the rest stay stale +// and appear the next time round, once whatever they were waiting for is registered. A chain and a +// diamond need no special handling, and nothing holds a plan. + +import { on, emit } from "@novox/mesh-sdk/events"; +import { Graph, type BuiltAgainst } from "./store.js"; + +const graph = Graph.fromEnv(); + +// Before subscribing, and idempotent. The runtime is restarted until its store is reachable, which +// is the same arrangement model-usage uses: a schema step that had to reach the provider over the +// overlay would block the very apply that brings the overlay up. +await graph.migrate(); + +/** What the builder says when it has built something. */ +interface Built { + module?: string; + commit?: string; + repository?: string; + path?: string; + ref?: string; + manifest?: unknown; + /** Every artifact this was built against, so the edge is derived rather than declared. */ + against?: { module: string; commit: string }[]; +} + +await on("module.builder.built", async (event) => { + const body = event.body as Built; + if (!body.module || !body.commit) { + // Said rather than dropped: a build that announced itself without saying what it built is a + // fault in the builder, and a silent discard here would make it look like a missing event. + console.error("mesh-catalog: a build event named no module or no commit; ignored", body); + return; + } + + const against: BuiltAgainst[] = (body.against ?? []).map((a) => ({ + module: body.module as string, + commit: body.commit as string, + againstModule: a.module, + againstCommit: a.commit, + })); + + const { upgraded, previous } = await graph.register({ + module: body.module, + commit: body.commit, + repository: body.repository ?? "", + path: body.path ?? "", + ref: body.ref ?? "", + manifest: body.manifest ?? {}, + }, against); + + await emit("module.mesh-catalog.registered", { + module: body.module, commit: body.commit, upgraded, + }); + + // **A rebuild that changed nothing is not an upgrade.** Announcing it would ripple outward + // through modules that did not change, forever (ADR 0072). + if (!upgraded) return; + + await emit("module.mesh-catalog.upgraded", { + module: body.module, commit: body.commit, previous, + }); + + // What can be built now — stale, and waiting on nothing that is itself stale. + for (const next of await graph.buildable()) { + await emit("module.mesh-catalog.rebuild-needed", { + module: next.module, + builtAt: next.commit, + because: next.because, + }); + } +}); diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json new file mode 100644 index 0000000..8ed36a3 --- /dev/null +++ b/modules/mesh-catalog/module.json @@ -0,0 +1,77 @@ +{ + "module": "mesh-catalog", + "slug": "catalog", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "claims": [ + { + "name": "the-catalogue", + "scope": "mesh" + } + ], + "requires": [ + "postgres-database" + ], + "contributes": { + "postgres-database": { + "name": "mesh_catalog" + } + }, + "binds": { + "postgres-database": "/var/lib/mesh-catalog/database.json" + }, + "secrets": { + "postgres-database": "/var/lib/mesh-catalog/database.secret" + }, + "own-secrets": { + "broker": "/var/lib/mesh/mesh-catalog/broker" + }, + "consumes": [ + "module.builder.built" + ], + "emits": [ + "module.mesh-catalog.registered", + "module.mesh-catalog.upgraded", + "module.mesh-catalog.rebuild-needed" + ], + "resources": [ + { + "id": "mesh-state", + "type": "directory", + "path": "/var/lib/mesh/mesh-catalog", + "mode": "0700" + }, + { + "id": "state", + "type": "directory", + "path": "/var/lib/mesh-catalog", + "mode": "0700" + }, + { + "id": "db-env", + "type": "file", + "path": "/var/lib/mesh-catalog/db.env", + "mode": "0600", + "content": "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + }, + { + "id": "runtime", + "type": "container", + "name": "mesh-catalog", + "image": "mesh-runtime-mesh-catalog@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "network": "host", + "volumes": [ + "/var/lib/mesh/mesh-catalog/broker:/run/secrets/broker:ro", + "/var/lib/mesh-catalog:/run/state" + ], + "env": { + "MESH_BROKER_FILE": "/run/secrets/broker" + }, + "env-file": [ + "/var/lib/mesh-catalog/db.env" + ] + } + ] +} diff --git a/modules/mesh-catalog/package.json b/modules/mesh-catalog/package.json new file mode 100644 index 0000000..3b60549 --- /dev/null +++ b/modules/mesh-catalog/package.json @@ -0,0 +1,15 @@ +{ + "name": "@novox/module-mesh-catalog", + "version": "0.1.0", + "description": "mesh-catalog — the module graph (novox/hq ADR 0070, 0072): links module-versions to each other, registers what the builder announces, and says what a change reaches and what can be rebuilt now. Knows nothing about nodes.", + "type": "module", + "private": true, + "dependencies": { + "@novox/mesh-sdk": "^0.1.0", + "pg": "^8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.6.0" + } +} diff --git a/modules/mesh-catalog/pg.d.ts b/modules/mesh-catalog/pg.d.ts new file mode 100644 index 0000000..023b330 --- /dev/null +++ b/modules/mesh-catalog/pg.d.ts @@ -0,0 +1,22 @@ +// Ambient types for `pg` (node-postgres), which ships its types only via the separate `@types/pg` +// package. Rather than pull that in at tsc time, this declares the exact slice model-usage uses — +// the same precedent anthropic-manager sets for `tweetnacl-sealedbox-js` (a local ambient .d.ts, +// listed in tsconfig `include`, default-imported). The real `pg` is installed into the module's +// runtime image (package.json `dependencies`; novox/hq ADR 0052), so this types the code without +// deciding what runs. +declare module "pg" { + /** One checked-out connection. Needed because registering a module-version and its edges is one + * act: a half-written registration is a graph that lies about what something was built against. */ + export class PoolClient { + query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>; + release(): void; + } + /** A lazily-connecting connection pool. Only the connection string, one query form, checking out + * a connection, and end() are used here. */ + export class Pool { + constructor(config?: { connectionString?: string }); + query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>; + connect(): Promise; + end(): Promise; + } +} diff --git a/modules/mesh-catalog/store.ts b/modules/mesh-catalog/store.ts new file mode 100644 index 0000000..7863acd --- /dev/null +++ b/modules/mesh-catalog/store.ts @@ -0,0 +1,236 @@ +// The module graph (novox/hq ADR 0070, ADR 0072). +// +// **This graph links module-versions to each other and knows nothing about nodes.** Which machine +// runs what is the control plane's graph, and the two meet only when somebody installs something. +// Keeping them apart is what lets the control plane carry on composing declarations while this is +// down: it holds what it needs, and asks nothing here. +// +// A module-version is identified by its commit, not by a version string. A version is what somebody +// calls a release; a commit is what was actually built (ADR 0009). + +import pg from "pg"; + +// `pg` is CommonJS: its default export is the module object, so Pool is a property of it. This is +// the form that resolves under Node's ESM loader — the same shape model-usage uses. +const { Pool } = pg; +type PgPool = InstanceType; + +/** One module-version, as the builder delivered it. */ +export interface ModuleVersion { + module: string; + commit: string; + repository: string; + /** The module's directory inside that repository (novox/hq ADR 0069). Empty is the root. */ + path: string; + ref: string; + /** The manifest with its artifacts pinned — the module as the mesh should hold it. */ + manifest: unknown; +} + +/** An edge: this module-version was built against that one. Derived, never declared (ADR 0009). */ +export interface BuiltAgainst { + module: string; + commit: string; + againstModule: string; + againstCommit: string; +} + +/** A module whose artifacts were built against something that is no longer current. */ +export interface Stale { + module: string; + commit: string; + /** What moved underneath it, and to where. */ + because: { module: string; builtAgainst: string; nowAt: string }[]; +} + +const DDL = ` +CREATE TABLE IF NOT EXISTS module_version ( + module text NOT NULL, + commit_sha text NOT NULL, + repository text NOT NULL DEFAULT '', + path text NOT NULL DEFAULT '', + ref text NOT NULL DEFAULT '', + manifest jsonb NOT NULL DEFAULT '{}'::jsonb, + registered timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (module, commit_sha) +); + +-- What the catalogue considers this module to be now. Separate from module_version because a +-- module has many versions and exactly one current one, and "current" is a decision rather than +-- a consequence of being newest. +CREATE TABLE IF NOT EXISTS module_current ( + module text PRIMARY KEY, + commit_sha text NOT NULL, + moved timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS built_against ( + module text NOT NULL, + commit_sha text NOT NULL, + against_module text NOT NULL, + against_commit text NOT NULL, + PRIMARY KEY (module, commit_sha, against_module, against_commit) +); + +CREATE INDEX IF NOT EXISTS built_against_target ON built_against (against_module); +`; + +export class Graph { + private constructor(private readonly pool: PgPool) {} + + static fromEnv(env: NodeJS.ProcessEnv = process.env): Graph { + const url = env["DATABASE_URL"]; + if (!url) { + throw new Error( + "no DATABASE_URL: the catalogue holds the module graph and cannot hold it in memory, " + + "because a graph that disappears on restart is not a record of anything", + ); + } + return new Graph(new Pool({ connectionString: url })); + } + + async migrate(): Promise { + await this.pool.query(DDL); + } + + /** + * Take a module-version the builder produced, and place it in the graph. + * + * **Returns whether this is an upgrade**, which is not the same as whether a build happened. A + * rebuild producing the commit already current changes nothing, and announcing it as an upgrade + * would ripple outward forever through modules that did not change (ADR 0072). + */ + async register(version: ModuleVersion, against: BuiltAgainst[]): Promise<{ upgraded: boolean; previous: string | null }> { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query( + `INSERT INTO module_version (module, commit_sha, repository, path, ref, manifest) + VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (module, commit_sha) DO UPDATE SET + repository = excluded.repository, path = excluded.path, + ref = excluded.ref, manifest = excluded.manifest`, + [version.module, version.commit, version.repository, version.path, version.ref, + JSON.stringify(version.manifest ?? {})], + ); + + // The edges are replaced rather than added to: they describe this build, and a previous + // build of the same commit that saw different dependencies was wrong about one of them. + await client.query(`DELETE FROM built_against WHERE module = $1 AND commit_sha = $2`, + [version.module, version.commit]); + for (const e of against) { + await client.query( + `INSERT INTO built_against (module, commit_sha, against_module, against_commit) + VALUES ($1,$2,$3,$4) ON CONFLICT DO NOTHING`, + [version.module, version.commit, e.againstModule, e.againstCommit]); + } + + const was = await client.query( + `SELECT commit_sha FROM module_current WHERE module = $1`, [version.module]); + const previous = (was.rows[0]?.commit_sha as string | undefined) ?? null; + const upgraded = previous !== version.commit; + if (upgraded) { + await client.query( + `INSERT INTO module_current (module, commit_sha) VALUES ($1,$2) + ON CONFLICT (module) DO UPDATE SET commit_sha = excluded.commit_sha, moved = now()`, + [version.module, version.commit]); + } + await client.query("COMMIT"); + return { upgraded, previous }; + } catch (err) { + await this.pool.query("ROLLBACK").catch(() => {}); + throw err; + } finally { + client.release(); + } + } + + /** Every module the catalogue holds, with the commit it considers current. */ + async modules(): Promise<{ module: string; commit: string; repository: string; path: string }[]> { + const { rows } = await this.pool.query( + `SELECT c.module, c.commit_sha, v.repository, v.path + FROM module_current c + JOIN module_version v ON v.module = c.module AND v.commit_sha = c.commit_sha + ORDER BY c.module`); + return rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, + repository: r.repository as string, path: r.path as string, + })); + } + + /** One module-version in full, current unless a commit is named. */ + async show(module: string, commit?: string): Promise { + const { rows } = await this.pool.query( + commit + ? `SELECT * FROM module_version WHERE module = $1 AND commit_sha = $2` + : `SELECT v.* FROM module_version v JOIN module_current c + ON c.module = v.module AND c.commit_sha = v.commit_sha WHERE v.module = $1`, + commit ? [module, commit] : [module]); + const r = rows[0]; + if (!r) return null; + return { + module: r.module as string, commit: r.commit_sha as string, + repository: r.repository as string, path: r.path as string, + ref: r.ref as string, manifest: r.manifest, + }; + } + + /** What was built against this module — the modules a change to it reaches. */ + async dependents(module: string): Promise<{ module: string; commit: string; againstCommit: string }[]> { + const { rows } = await this.pool.query( + `SELECT b.module, b.commit_sha, b.against_commit + FROM built_against b + JOIN module_current c ON c.module = b.module AND c.commit_sha = b.commit_sha + WHERE b.against_module = $1 + ORDER BY b.module`, [module]); + return rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, + againstCommit: r.against_commit as string, + })); + } + + /** + * What must be rebuilt, and why. + * + * **The rule is a condition, not an order** (ADR 0072): a module is stale when anything it was + * built against is no longer current. Asking this repeatedly is what produces the right order — + * a module whose own dependencies are still stale simply stays stale until they are not, so a + * chain and a diamond need no special handling and nothing has to know the shape in advance. + */ + async stale(): Promise { + const { rows } = await this.pool.query( + `SELECT b.module, b.commit_sha, b.against_module, b.against_commit, c2.commit_sha AS now_at + FROM built_against b + JOIN module_current c1 ON c1.module = b.module AND c1.commit_sha = b.commit_sha + JOIN module_current c2 ON c2.module = b.against_module + WHERE c2.commit_sha <> b.against_commit + ORDER BY b.module`); + const by = new Map(); + for (const r of rows) { + const key = `${r.module}@${r.commit_sha}`; + const entry = by.get(key) ?? { module: r.module as string, commit: r.commit_sha as string, because: [] }; + entry.because.push({ + module: r.against_module as string, + builtAgainst: r.against_commit as string, + nowAt: r.now_at as string, + }); + by.set(key, entry); + } + return [...by.values()]; + } + + /** + * Modules that are stale and whose own dependencies are all current — the ones that can be built + * now. **This is the whole of ordering.** Anything not in this set is waiting for something else, + * and will appear here once that thing is registered. + */ + async buildable(): Promise { + const stale = await this.stale(); + const waiting = new Set(stale.map((s) => s.module)); + return stale.filter((s) => !s.because.some((b) => waiting.has(b.module))); + } + + async close(): Promise { + await this.pool.end(); + } +} diff --git a/modules/mesh-catalog/tools/index.ts b/modules/mesh-catalog/tools/index.ts new file mode 100644 index 0000000..09a1cba --- /dev/null +++ b/modules/mesh-catalog/tools/index.ts @@ -0,0 +1,67 @@ +// mesh-catalog's tools — the module graph's query surface (novox/hq ADR 0070). +// +// These are the questions the graph exists to answer, and none of them can be answered anywhere +// else today: what does this mesh know how to run, what is this module made of, what does a change +// to this reach, and what is waiting to be rebuilt. + +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; +import { Graph } from "../store.js"; + +export function getCatalogueTools(graph: Graph): ToolDefinition[] { + return [ + { + name: "catalog_modules", + description: + "Every module this mesh holds, with the commit the catalogue considers current and where it came from.", + input: {}, + run: async () => ({ modules: await graph.modules() }), + }, + { + name: "catalog_module", + description: + "One module in full — where it came from, what it requires and provides, and what it is made of. The current version unless a commit is named.", + input: { + module: { type: "string", description: "the module's name" }, + commit: { type: "string", description: "a particular version (optional)" }, + }, + run: async (args) => { + const module = String(args.module ?? ""); + if (!module) return { error: "name a module" }; + const found = await graph.show(module, args.commit ? String(args.commit) : undefined); + return found ? { module: found } : { error: `the catalogue holds no ${module}` }; + }, + }, + { + name: "catalog_dependents", + description: + "What was built against this module — the modules a change to it reaches. Derived from builds, not from a declared list, so it cannot drift from what the code actually uses.", + input: { module: { type: "string", description: "the module that would change" } }, + run: async (args) => { + const module = String(args.module ?? ""); + if (!module) return { error: "name a module" }; + return { module, dependents: await graph.dependents(module) }; + }, + }, + { + name: "catalog_stale", + description: + "What must be rebuilt and why — every module built against something that has since moved. `buildable` is the subset waiting on nothing that is itself stale, which is the set that can be built right now.", + input: {}, + run: async () => ({ + stale: await graph.stale(), + buildable: await graph.buildable(), + }), + }, + ]; +} + +// Opened from the environment when the runtime asks for the module's tools. When it cannot be — +// no DATABASE_URL — the module contributes no tools rather than taking the whole tool runtime down +// with it, which is the postgres precedent. +registerModuleTools("mesh-catalog", (env) => { + try { + return getCatalogueTools(Graph.fromEnv(env)); + } catch { + return []; + } +}); diff --git a/modules/mesh-catalog/tsconfig.json b/modules/mesh-catalog/tsconfig.json new file mode 100644 index 0000000..2528f05 --- /dev/null +++ b/modules/mesh-catalog/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": [ + "pg.d.ts", + "store.ts", + "index.ts", + "tools/index.ts" + ] +}