diff --git a/modules/mesh-catalog/index.ts b/modules/mesh-catalog/index.ts new file mode 100644 index 0000000..7d7f29d --- /dev/null +++ b/modules/mesh-catalog/index.ts @@ -0,0 +1,79 @@ +// mesh-catalog's entrypoint — the module graph's consumer (novox/hq ADR 0070, ADR 0072). +// +// The builder announces what it built; this places it in the graph and announces what that means. +// The control plane hooks the *meaning* — a module was upgraded — rather than the build output, so +// it never has to interpret an artifact or ask this module anything. +// +// **Ordering is not computed here.** When a registration makes something else stale, the modules +// whose own dependencies are all current are announced as needing a rebuild; the rest stay stale +// and appear the next time round, once whatever they were waiting for is registered. A chain and a +// diamond need no special handling, and nothing holds a plan. + +import { on, emit } from "@novox/mesh-sdk/events"; +import { Graph, type BuiltAgainst } from "./store.js"; + +const graph = Graph.fromEnv(); + +// Before subscribing, and idempotent. The runtime is restarted until its store is reachable, which +// is the same arrangement model-usage uses: a schema step that had to reach the provider over the +// overlay would block the very apply that brings the overlay up. +await graph.migrate(); + +/** What the builder says when it has built something. */ +interface Built { + module?: string; + commit?: string; + repository?: string; + path?: string; + ref?: string; + manifest?: unknown; + /** Every artifact this was built against, so the edge is derived rather than declared. */ + against?: { module: string; commit: string }[]; +} + +await on("module.builder.built", async (event) => { + const body = event.body as Built; + if (!body.module || !body.commit) { + // Said rather than dropped: a build that announced itself without saying what it built is a + // fault in the builder, and a silent discard here would make it look like a missing event. + console.error("mesh-catalog: a build event named no module or no commit; ignored", body); + return; + } + + const against: BuiltAgainst[] = (body.against ?? []).map((a) => ({ + module: body.module as string, + commit: body.commit as string, + againstModule: a.module, + againstCommit: a.commit, + })); + + const { upgraded, previous } = await graph.register({ + module: body.module, + commit: body.commit, + repository: body.repository ?? "", + path: body.path ?? "", + ref: body.ref ?? "", + manifest: body.manifest ?? {}, + }, against); + + await emit("module.mesh-catalog.registered", { + module: body.module, commit: body.commit, upgraded, + }); + + // **A rebuild that changed nothing is not an upgrade.** Announcing it would ripple outward + // through modules that did not change, forever (ADR 0072). + if (!upgraded) return; + + await emit("module.mesh-catalog.upgraded", { + module: body.module, commit: body.commit, previous, + }); + + // What can be built now — stale, and waiting on nothing that is itself stale. + for (const next of await graph.buildable()) { + await emit("module.mesh-catalog.rebuild-needed", { + module: next.module, + builtAt: next.commit, + because: next.because, + }); + } +}); diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json new file mode 100644 index 0000000..8ed36a3 --- /dev/null +++ b/modules/mesh-catalog/module.json @@ -0,0 +1,77 @@ +{ + "module": "mesh-catalog", + "slug": "catalog", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "claims": [ + { + "name": "the-catalogue", + "scope": "mesh" + } + ], + "requires": [ + "postgres-database" + ], + "contributes": { + "postgres-database": { + "name": "mesh_catalog" + } + }, + "binds": { + "postgres-database": "/var/lib/mesh-catalog/database.json" + }, + "secrets": { + "postgres-database": "/var/lib/mesh-catalog/database.secret" + }, + "own-secrets": { + "broker": "/var/lib/mesh/mesh-catalog/broker" + }, + "consumes": [ + "module.builder.built" + ], + "emits": [ + "module.mesh-catalog.registered", + "module.mesh-catalog.upgraded", + "module.mesh-catalog.rebuild-needed" + ], + "resources": [ + { + "id": "mesh-state", + "type": "directory", + "path": "/var/lib/mesh/mesh-catalog", + "mode": "0700" + }, + { + "id": "state", + "type": "directory", + "path": "/var/lib/mesh-catalog", + "mode": "0700" + }, + { + "id": "db-env", + "type": "file", + "path": "/var/lib/mesh-catalog/db.env", + "mode": "0600", + "content": "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + }, + { + "id": "runtime", + "type": "container", + "name": "mesh-catalog", + "image": "mesh-runtime-mesh-catalog@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "network": "host", + "volumes": [ + "/var/lib/mesh/mesh-catalog/broker:/run/secrets/broker:ro", + "/var/lib/mesh-catalog:/run/state" + ], + "env": { + "MESH_BROKER_FILE": "/run/secrets/broker" + }, + "env-file": [ + "/var/lib/mesh-catalog/db.env" + ] + } + ] +} diff --git a/modules/mesh-catalog/package.json b/modules/mesh-catalog/package.json new file mode 100644 index 0000000..3b60549 --- /dev/null +++ b/modules/mesh-catalog/package.json @@ -0,0 +1,15 @@ +{ + "name": "@novox/module-mesh-catalog", + "version": "0.1.0", + "description": "mesh-catalog — the module graph (novox/hq ADR 0070, 0072): links module-versions to each other, registers what the builder announces, and says what a change reaches and what can be rebuilt now. Knows nothing about nodes.", + "type": "module", + "private": true, + "dependencies": { + "@novox/mesh-sdk": "^0.1.0", + "pg": "^8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.6.0" + } +} diff --git a/modules/mesh-catalog/pg.d.ts b/modules/mesh-catalog/pg.d.ts new file mode 100644 index 0000000..023b330 --- /dev/null +++ b/modules/mesh-catalog/pg.d.ts @@ -0,0 +1,22 @@ +// Ambient types for `pg` (node-postgres), which ships its types only via the separate `@types/pg` +// package. Rather than pull that in at tsc time, this declares the exact slice model-usage uses — +// the same precedent anthropic-manager sets for `tweetnacl-sealedbox-js` (a local ambient .d.ts, +// listed in tsconfig `include`, default-imported). The real `pg` is installed into the module's +// runtime image (package.json `dependencies`; novox/hq ADR 0052), so this types the code without +// deciding what runs. +declare module "pg" { + /** One checked-out connection. Needed because registering a module-version and its edges is one + * act: a half-written registration is a graph that lies about what something was built against. */ + export class PoolClient { + query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>; + release(): void; + } + /** A lazily-connecting connection pool. Only the connection string, one query form, checking out + * a connection, and end() are used here. */ + export class Pool { + constructor(config?: { connectionString?: string }); + query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>; + connect(): Promise; + end(): Promise; + } +} diff --git a/modules/mesh-catalog/store.ts b/modules/mesh-catalog/store.ts new file mode 100644 index 0000000..7863acd --- /dev/null +++ b/modules/mesh-catalog/store.ts @@ -0,0 +1,236 @@ +// The module graph (novox/hq ADR 0070, ADR 0072). +// +// **This graph links module-versions to each other and knows nothing about nodes.** Which machine +// runs what is the control plane's graph, and the two meet only when somebody installs something. +// Keeping them apart is what lets the control plane carry on composing declarations while this is +// down: it holds what it needs, and asks nothing here. +// +// A module-version is identified by its commit, not by a version string. A version is what somebody +// calls a release; a commit is what was actually built (ADR 0009). + +import pg from "pg"; + +// `pg` is CommonJS: its default export is the module object, so Pool is a property of it. This is +// the form that resolves under Node's ESM loader — the same shape model-usage uses. +const { Pool } = pg; +type PgPool = InstanceType; + +/** One module-version, as the builder delivered it. */ +export interface ModuleVersion { + module: string; + commit: string; + repository: string; + /** The module's directory inside that repository (novox/hq ADR 0069). Empty is the root. */ + path: string; + ref: string; + /** The manifest with its artifacts pinned — the module as the mesh should hold it. */ + manifest: unknown; +} + +/** An edge: this module-version was built against that one. Derived, never declared (ADR 0009). */ +export interface BuiltAgainst { + module: string; + commit: string; + againstModule: string; + againstCommit: string; +} + +/** A module whose artifacts were built against something that is no longer current. */ +export interface Stale { + module: string; + commit: string; + /** What moved underneath it, and to where. */ + because: { module: string; builtAgainst: string; nowAt: string }[]; +} + +const DDL = ` +CREATE TABLE IF NOT EXISTS module_version ( + module text NOT NULL, + commit_sha text NOT NULL, + repository text NOT NULL DEFAULT '', + path text NOT NULL DEFAULT '', + ref text NOT NULL DEFAULT '', + manifest jsonb NOT NULL DEFAULT '{}'::jsonb, + registered timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (module, commit_sha) +); + +-- What the catalogue considers this module to be now. Separate from module_version because a +-- module has many versions and exactly one current one, and "current" is a decision rather than +-- a consequence of being newest. +CREATE TABLE IF NOT EXISTS module_current ( + module text PRIMARY KEY, + commit_sha text NOT NULL, + moved timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS built_against ( + module text NOT NULL, + commit_sha text NOT NULL, + against_module text NOT NULL, + against_commit text NOT NULL, + PRIMARY KEY (module, commit_sha, against_module, against_commit) +); + +CREATE INDEX IF NOT EXISTS built_against_target ON built_against (against_module); +`; + +export class Graph { + private constructor(private readonly pool: PgPool) {} + + static fromEnv(env: NodeJS.ProcessEnv = process.env): Graph { + const url = env["DATABASE_URL"]; + if (!url) { + throw new Error( + "no DATABASE_URL: the catalogue holds the module graph and cannot hold it in memory, " + + "because a graph that disappears on restart is not a record of anything", + ); + } + return new Graph(new Pool({ connectionString: url })); + } + + async migrate(): Promise { + await this.pool.query(DDL); + } + + /** + * Take a module-version the builder produced, and place it in the graph. + * + * **Returns whether this is an upgrade**, which is not the same as whether a build happened. A + * rebuild producing the commit already current changes nothing, and announcing it as an upgrade + * would ripple outward forever through modules that did not change (ADR 0072). + */ + async register(version: ModuleVersion, against: BuiltAgainst[]): Promise<{ upgraded: boolean; previous: string | null }> { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query( + `INSERT INTO module_version (module, commit_sha, repository, path, ref, manifest) + VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (module, commit_sha) DO UPDATE SET + repository = excluded.repository, path = excluded.path, + ref = excluded.ref, manifest = excluded.manifest`, + [version.module, version.commit, version.repository, version.path, version.ref, + JSON.stringify(version.manifest ?? {})], + ); + + // The edges are replaced rather than added to: they describe this build, and a previous + // build of the same commit that saw different dependencies was wrong about one of them. + await client.query(`DELETE FROM built_against WHERE module = $1 AND commit_sha = $2`, + [version.module, version.commit]); + for (const e of against) { + await client.query( + `INSERT INTO built_against (module, commit_sha, against_module, against_commit) + VALUES ($1,$2,$3,$4) ON CONFLICT DO NOTHING`, + [version.module, version.commit, e.againstModule, e.againstCommit]); + } + + const was = await client.query( + `SELECT commit_sha FROM module_current WHERE module = $1`, [version.module]); + const previous = (was.rows[0]?.commit_sha as string | undefined) ?? null; + const upgraded = previous !== version.commit; + if (upgraded) { + await client.query( + `INSERT INTO module_current (module, commit_sha) VALUES ($1,$2) + ON CONFLICT (module) DO UPDATE SET commit_sha = excluded.commit_sha, moved = now()`, + [version.module, version.commit]); + } + await client.query("COMMIT"); + return { upgraded, previous }; + } catch (err) { + await this.pool.query("ROLLBACK").catch(() => {}); + throw err; + } finally { + client.release(); + } + } + + /** Every module the catalogue holds, with the commit it considers current. */ + async modules(): Promise<{ module: string; commit: string; repository: string; path: string }[]> { + const { rows } = await this.pool.query( + `SELECT c.module, c.commit_sha, v.repository, v.path + FROM module_current c + JOIN module_version v ON v.module = c.module AND v.commit_sha = c.commit_sha + ORDER BY c.module`); + return rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, + repository: r.repository as string, path: r.path as string, + })); + } + + /** One module-version in full, current unless a commit is named. */ + async show(module: string, commit?: string): Promise { + const { rows } = await this.pool.query( + commit + ? `SELECT * FROM module_version WHERE module = $1 AND commit_sha = $2` + : `SELECT v.* FROM module_version v JOIN module_current c + ON c.module = v.module AND c.commit_sha = v.commit_sha WHERE v.module = $1`, + commit ? [module, commit] : [module]); + const r = rows[0]; + if (!r) return null; + return { + module: r.module as string, commit: r.commit_sha as string, + repository: r.repository as string, path: r.path as string, + ref: r.ref as string, manifest: r.manifest, + }; + } + + /** What was built against this module — the modules a change to it reaches. */ + async dependents(module: string): Promise<{ module: string; commit: string; againstCommit: string }[]> { + const { rows } = await this.pool.query( + `SELECT b.module, b.commit_sha, b.against_commit + FROM built_against b + JOIN module_current c ON c.module = b.module AND c.commit_sha = b.commit_sha + WHERE b.against_module = $1 + ORDER BY b.module`, [module]); + return rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, + againstCommit: r.against_commit as string, + })); + } + + /** + * What must be rebuilt, and why. + * + * **The rule is a condition, not an order** (ADR 0072): a module is stale when anything it was + * built against is no longer current. Asking this repeatedly is what produces the right order — + * a module whose own dependencies are still stale simply stays stale until they are not, so a + * chain and a diamond need no special handling and nothing has to know the shape in advance. + */ + async stale(): Promise { + const { rows } = await this.pool.query( + `SELECT b.module, b.commit_sha, b.against_module, b.against_commit, c2.commit_sha AS now_at + FROM built_against b + JOIN module_current c1 ON c1.module = b.module AND c1.commit_sha = b.commit_sha + JOIN module_current c2 ON c2.module = b.against_module + WHERE c2.commit_sha <> b.against_commit + ORDER BY b.module`); + const by = new Map(); + for (const r of rows) { + const key = `${r.module}@${r.commit_sha}`; + const entry = by.get(key) ?? { module: r.module as string, commit: r.commit_sha as string, because: [] }; + entry.because.push({ + module: r.against_module as string, + builtAgainst: r.against_commit as string, + nowAt: r.now_at as string, + }); + by.set(key, entry); + } + return [...by.values()]; + } + + /** + * Modules that are stale and whose own dependencies are all current — the ones that can be built + * now. **This is the whole of ordering.** Anything not in this set is waiting for something else, + * and will appear here once that thing is registered. + */ + async buildable(): Promise { + const stale = await this.stale(); + const waiting = new Set(stale.map((s) => s.module)); + return stale.filter((s) => !s.because.some((b) => waiting.has(b.module))); + } + + async close(): Promise { + await this.pool.end(); + } +} diff --git a/modules/mesh-catalog/tools/index.ts b/modules/mesh-catalog/tools/index.ts new file mode 100644 index 0000000..09a1cba --- /dev/null +++ b/modules/mesh-catalog/tools/index.ts @@ -0,0 +1,67 @@ +// mesh-catalog's tools — the module graph's query surface (novox/hq ADR 0070). +// +// These are the questions the graph exists to answer, and none of them can be answered anywhere +// else today: what does this mesh know how to run, what is this module made of, what does a change +// to this reach, and what is waiting to be rebuilt. + +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; +import { Graph } from "../store.js"; + +export function getCatalogueTools(graph: Graph): ToolDefinition[] { + return [ + { + name: "catalog_modules", + description: + "Every module this mesh holds, with the commit the catalogue considers current and where it came from.", + input: {}, + run: async () => ({ modules: await graph.modules() }), + }, + { + name: "catalog_module", + description: + "One module in full — where it came from, what it requires and provides, and what it is made of. The current version unless a commit is named.", + input: { + module: { type: "string", description: "the module's name" }, + commit: { type: "string", description: "a particular version (optional)" }, + }, + run: async (args) => { + const module = String(args.module ?? ""); + if (!module) return { error: "name a module" }; + const found = await graph.show(module, args.commit ? String(args.commit) : undefined); + return found ? { module: found } : { error: `the catalogue holds no ${module}` }; + }, + }, + { + name: "catalog_dependents", + description: + "What was built against this module — the modules a change to it reaches. Derived from builds, not from a declared list, so it cannot drift from what the code actually uses.", + input: { module: { type: "string", description: "the module that would change" } }, + run: async (args) => { + const module = String(args.module ?? ""); + if (!module) return { error: "name a module" }; + return { module, dependents: await graph.dependents(module) }; + }, + }, + { + name: "catalog_stale", + description: + "What must be rebuilt and why — every module built against something that has since moved. `buildable` is the subset waiting on nothing that is itself stale, which is the set that can be built right now.", + input: {}, + run: async () => ({ + stale: await graph.stale(), + buildable: await graph.buildable(), + }), + }, + ]; +} + +// Opened from the environment when the runtime asks for the module's tools. When it cannot be — +// no DATABASE_URL — the module contributes no tools rather than taking the whole tool runtime down +// with it, which is the postgres precedent. +registerModuleTools("mesh-catalog", (env) => { + try { + return getCatalogueTools(Graph.fromEnv(env)); + } catch { + return []; + } +}); diff --git a/modules/mesh-catalog/tsconfig.json b/modules/mesh-catalog/tsconfig.json new file mode 100644 index 0000000..2528f05 --- /dev/null +++ b/modules/mesh-catalog/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": [ + "pg.d.ts", + "store.ts", + "index.ts", + "tools/index.ts" + ] +}