diff --git a/modules/amqp-ping/Dockerfile b/modules/amqp-ping/Dockerfile new file mode 100644 index 0000000..3f4a7b1 --- /dev/null +++ b/modules/amqp-ping/Dockerfile @@ -0,0 +1,26 @@ +# amqp-ping's runtime: the tool runtime, carrying this module's compiled code. +# +# **Built from this module's own directory and nothing else.** The sdk and the tool runtime are not +# copied out of neighbouring checkouts — they are in the base image, which is published like any +# other artifact. That is what makes this buildable by the mesh from a repository and a path +# (novox/hq ADR 0069) rather than only on a workstation that happens to have the siblings. +# +# The base is named by ARG so it can be pinned to a digest the mesh's registry assigned. A tag here +# would make the runtime's contents depend on what somebody last pushed under that name. +ARG RUNTIME_BASE=127.0.0.1:5000/mesh-tools/runtime@sha256:44b5d8bc30107fdc3bffdaebc1ca2de97615053853f826dfccce253a01a163fd + +FROM ${RUNTIME_BASE} AS build +# Compiled under /app/modules, so resolving `@novox/mesh-sdk` walks up to the base's own +# node_modules — the module is compiled against exactly the sdk it will run against. +WORKDIR /app/modules/amqp-ping +COPY . . +# The compiler is invoked by its real path, not through node_modules/.bin. Those are symlinks to +# a launcher that requires its library relatively, and the base image resolves them when copying — +# leaving a launcher whose relative require no longer points at anything. +RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts \ + --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist + +FROM ${RUNTIME_BASE} +COPY --from=build /app/modules/amqp-ping/dist /app/modules/amqp-ping/dist +# Declared rather than derived from which files happen to exist: the module knows what it serves. +ENV MESH_TOOL_MODULES=/app/modules/amqp-ping/dist/index.js diff --git a/modules/amqp-ping/module.json b/modules/amqp-ping/module.json index ab4a0cb..5e7184c 100644 --- a/modules/amqp-ping/module.json +++ b/modules/amqp-ping/module.json @@ -47,18 +47,23 @@ "id": "runtime", "type": "container", "name": "amqp-ping", - "image": "mesh-runtime-amqp-ping@sha256:0000000000000000000000000000000000000000000000000000000000000000", "network": "amqp-ping", "env-file": [ "/var/lib/amqp-ping/amqp.env" ], - "args": [ - "run", - "/app/modules/amqp-ping/dist/index.js" - ], "restart-on": [ "amqp-env" - ] + ], + "artifact": "runtime" } - ] + ], + "build": { + "artifacts": [ + { + "name": "runtime", + "kind": "image", + "from": "Dockerfile" + } + ] + } } diff --git a/modules/builder/module.json b/modules/builder/module.json index 8de43ea..d061ed7 100644 --- a/modules/builder/module.json +++ b/modules/builder/module.json @@ -13,6 +13,9 @@ "requires": [ "artifact-store" ], + "emits": [ + "module.builder.built" + ], "own-secrets": { "broker": "/var/lib/mesh/builder/broker" }, @@ -34,7 +37,7 @@ "type": "file", "path": "/var/lib/mesh/builder/builder.env", "mode": "0600", - "content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_REGISTRY=127.0.0.1:${bound:artifact-store:port}\nMESH_WORKSPACE=/workspace\n" + "content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=127.0.0.1:${bound:artifact-store:port}\nMESH_WORKSPACE=/workspace\n" }, { "id": "server", diff --git a/modules/hello-web/module.json b/modules/hello-web/module.json index fb2dfc3..177124a 100644 --- a/modules/hello-web/module.json +++ b/modules/hello-web/module.json @@ -48,7 +48,6 @@ "id": "server", "type": "container", "name": "hello-web", - "image": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b", "network": "hello-web", "ports": [ "8080:8080" @@ -60,7 +59,17 @@ "sh", "-c", "while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done" - ] + ], + "artifact": "server" } - ] + ], + "build": { + "artifacts": [ + { + "name": "server", + "kind": "upstream", + "from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + } + ] + } } diff --git a/modules/mesh-catalog/Dockerfile b/modules/mesh-catalog/Dockerfile new file mode 100644 index 0000000..5331f6b --- /dev/null +++ b/modules/mesh-catalog/Dockerfile @@ -0,0 +1,45 @@ +# mesh-catalog's runtime: the tool runtime, carrying the catalogue's compiled graph, its consumer +# of what the builder announces, and its tools. +# +# **Built from this module's own directory and nothing else.** The sdk is in the base image, so +# nothing is copied out of a neighbouring checkout — which is what lets the mesh build this from a +# repository and a path (novox/hq ADR 0069) rather than only on a workstation with the siblings. +# +# The base is named by ARG so it can be pinned to a digest the mesh's registry assigned. A tag would +# make this runtime's contents depend on what somebody last pushed under that name. +ARG RUNTIME_BASE=127.0.0.1:5000/mesh-tools/runtime@sha256:44b5d8bc30107fdc3bffdaebc1ca2de97615053853f826dfccce253a01a163fd + +FROM ${RUNTIME_BASE} AS build +# Compiled under /app/modules so `@novox/mesh-sdk` resolves upward into the base's own +# node_modules — the module is compiled against exactly the sdk it will run against. +WORKDIR /app/modules/mesh-catalog +COPY . . +# The compiler is invoked by its real path rather than through node_modules/.bin, whose entries are +# symlinks to a launcher that requires its library relatively — resolved away when the base image +# was assembled. +RUN node /app/node_modules/typescript/bin/tsc pg.d.ts store.ts index.ts tools/index.ts \ + --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist + +# **A module may need something the base image does not carry.** The base holds what every module +# needs — the sdk, the broker client — and a postgres driver is not that: the one other module that +# reaches a database shells out to psql instead. So the catalogue brings its own. +# +# Installed into an empty directory rather than into the module's, because the module's package.json +# also names `@novox/mesh-sdk`, which is not on any registry — it is in the base image. Asking npm to +# resolve this module's dependencies would therefore fail on the one it already has. +RUN mkdir -p /deps && cd /deps && \ + npm install --omit=dev --no-audit --no-fund --no-package-lock pg@8 + +FROM ${RUNTIME_BASE} +COPY --from=build /app/modules/mesh-catalog/dist /app/modules/mesh-catalog/dist +# Beside the compiled code, so `pg` resolves from it while `@novox/mesh-sdk` keeps walking up to the +# base image's own node_modules — the module gets its extra dependency without shadowing the sdk it +# was compiled against. +COPY --from=build /deps/node_modules /app/modules/mesh-catalog/node_modules +# Both entrypoints, loaded in serve mode. +# +# **A consumer cannot be started with `run`.** That mode imports an entrypoint without binding a +# broker — it is for a step that does its work offline and exits — and the catalogue's whole job is +# to listen for what the builder announces. Serve binds the broker first, then imports these, so +# `on()` has something to subscribe to. +ENV MESH_TOOL_MODULES=/app/modules/mesh-catalog/dist/index.js,/app/modules/mesh-catalog/dist/tools/index.js diff --git a/modules/mesh-catalog/index.ts b/modules/mesh-catalog/index.ts new file mode 100644 index 0000000..20c64f0 --- /dev/null +++ b/modules/mesh-catalog/index.ts @@ -0,0 +1,79 @@ +// mesh-catalog's entrypoint — the module graph's consumer (novox/hq ADR 0070, ADR 0072). +// +// The builder announces what it built; this places it in the graph and announces what that means. +// The control plane hooks the *meaning* — a module was upgraded — rather than the build output, so +// it never has to interpret an artifact or ask this module anything. +// +// **Ordering is not computed here.** When a registration makes something else stale, the modules +// whose own dependencies are all current are announced as needing a rebuild; the rest stay stale +// and appear the next time round, once whatever they were waiting for is registered. A chain and a +// diamond need no special handling, and nothing holds a plan. + +import { on, emit } from "@novox/mesh-sdk/events"; +import { Graph, type Made } from "./store.js"; + +const graph = Graph.fromEnv(); + +// Before subscribing, and idempotent. The runtime is restarted until its store is reachable, which +// is the same arrangement model-usage uses: a schema step that had to reach the provider over the +// overlay would block the very apply that brings the overlay up. +await graph.migrate(); + +/** What the builder says when it has built something. */ +interface Built { + module?: string; + commit?: string; + repository?: string; + path?: string; + ref?: string; + manifest?: unknown; + /** What this build published, each pinned as anything else would name it. */ + made?: Made[]; + /** + * Every artifact this was built on top of, so the edge is derived rather than declared. + * + * References, not module names: the builder sees a pinned image and cannot see which module + * produced it. Turning that into an edge between module-versions is this module's job. + */ + against?: string[]; +} + +await on("module.builder.built", async (event) => { + const body = event.body as Built; + if (!body.module || !body.commit) { + // Said rather than dropped: a build that announced itself without saying what it built is a + // fault in the builder, and a silent discard here would make it look like a missing event. + console.error("mesh-catalog: a build event named no module or no commit; ignored", body); + return; + } + + const { upgraded, previous } = await graph.register({ + module: body.module, + commit: body.commit, + repository: body.repository ?? "", + path: body.path ?? "", + ref: body.ref ?? "", + manifest: body.manifest ?? {}, + }, body.made ?? [], body.against ?? []); + + await emit("module.mesh-catalog.registered", { + module: body.module, commit: body.commit, upgraded, + }); + + // **A rebuild that changed nothing is not an upgrade.** Announcing it would ripple outward + // through modules that did not change, forever (ADR 0072). + if (!upgraded) return; + + await emit("module.mesh-catalog.upgraded", { + module: body.module, commit: body.commit, previous, + }); + + // What can be built now — stale, and waiting on nothing that is itself stale. + for (const next of await graph.buildable()) { + await emit("module.mesh-catalog.rebuild-needed", { + module: next.module, + builtAt: next.commit, + because: next.because, + }); + } +}); diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json new file mode 100644 index 0000000..6165f8b --- /dev/null +++ b/modules/mesh-catalog/module.json @@ -0,0 +1,89 @@ +{ + "module": "mesh-catalog", + "slug": "catalog", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "claims": [ + { + "name": "the-catalogue", + "scope": "mesh" + } + ], + "requires": [ + "postgres-database" + ], + "contributes": { + "postgres-database": { + "name": "mesh_catalog" + } + }, + "binds": { + "postgres-database": "/var/lib/mesh-catalog/database.json" + }, + "secrets": { + "postgres-database": "/var/lib/mesh-catalog/database.secret" + }, + "own-secrets": { + "broker": "/var/lib/mesh/mesh-catalog/broker" + }, + "consumes": [ + "module.builder.built" + ], + "emits": [ + "module.mesh-catalog.registered", + "module.mesh-catalog.upgraded", + "module.mesh-catalog.rebuild-needed" + ], + "resources": [ + { + "id": "mesh-state", + "type": "directory", + "path": "/var/lib/mesh/mesh-catalog", + "mode": "0700" + }, + { + "id": "state", + "type": "directory", + "path": "/var/lib/mesh-catalog", + "mode": "0700" + }, + { + "id": "db-env", + "type": "file", + "path": "/var/lib/mesh-catalog/db.env", + "mode": "0600", + "content": "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + }, + { + "id": "runtime", + "type": "container", + "name": "mesh-catalog", + "network": "host", + "volumes": [ + "/var/lib/mesh/mesh-catalog/broker:/run/secrets/broker:ro", + "/var/lib/mesh-catalog:/run/state" + ], + "env": { + "MESH_BROKER_FILE": "/run/secrets/broker" + }, + "env-file": [ + "/var/lib/mesh-catalog/db.env" + ], + "artifact": "runtime", + "restart-on": [ + "db-env" + ] + } + ], + "build": { + "artifacts": [ + { + "name": "runtime", + "kind": "image", + "from": "Dockerfile" + } + ] + } +} diff --git a/modules/mesh-catalog/package.json b/modules/mesh-catalog/package.json new file mode 100644 index 0000000..3b60549 --- /dev/null +++ b/modules/mesh-catalog/package.json @@ -0,0 +1,15 @@ +{ + "name": "@novox/module-mesh-catalog", + "version": "0.1.0", + "description": "mesh-catalog — the module graph (novox/hq ADR 0070, 0072): links module-versions to each other, registers what the builder announces, and says what a change reaches and what can be rebuilt now. Knows nothing about nodes.", + "type": "module", + "private": true, + "dependencies": { + "@novox/mesh-sdk": "^0.1.0", + "pg": "^8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.6.0" + } +} diff --git a/modules/mesh-catalog/pg.d.ts b/modules/mesh-catalog/pg.d.ts new file mode 100644 index 0000000..023b330 --- /dev/null +++ b/modules/mesh-catalog/pg.d.ts @@ -0,0 +1,22 @@ +// Ambient types for `pg` (node-postgres), which ships its types only via the separate `@types/pg` +// package. Rather than pull that in at tsc time, this declares the exact slice model-usage uses — +// the same precedent anthropic-manager sets for `tweetnacl-sealedbox-js` (a local ambient .d.ts, +// listed in tsconfig `include`, default-imported). The real `pg` is installed into the module's +// runtime image (package.json `dependencies`; novox/hq ADR 0052), so this types the code without +// deciding what runs. +declare module "pg" { + /** One checked-out connection. Needed because registering a module-version and its edges is one + * act: a half-written registration is a graph that lies about what something was built against. */ + export class PoolClient { + query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>; + release(): void; + } + /** A lazily-connecting connection pool. Only the connection string, one query form, checking out + * a connection, and end() are used here. */ + export class Pool { + constructor(config?: { connectionString?: string }); + query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>; + connect(): Promise; + end(): Promise; + } +} diff --git a/modules/mesh-catalog/store.ts b/modules/mesh-catalog/store.ts new file mode 100644 index 0000000..5b187ef --- /dev/null +++ b/modules/mesh-catalog/store.ts @@ -0,0 +1,392 @@ +// The module graph (novox/hq ADR 0070, ADR 0072). +// +// **This graph links module-versions to each other and knows nothing about nodes.** Which machine +// runs what is the control plane's graph, and the two meet only when somebody installs something. +// Keeping them apart is what lets the control plane carry on composing declarations while this is +// down: it holds what it needs, and asks nothing here. +// +// A module-version is identified by its commit, not by a version string. A version is what somebody +// calls a release; a commit is what was actually built (ADR 0009). + +import pg from "pg"; + +// `pg` is CommonJS: its default export is the module object, so Pool is a property of it. This is +// the form that resolves under Node's ESM loader — the same shape model-usage uses. +const { Pool } = pg; +type PgPool = InstanceType; + +/** One module-version, as the builder delivered it. */ +export interface ModuleVersion { + module: string; + commit: string; + repository: string; + /** The module's directory inside that repository (novox/hq ADR 0069). Empty is the root. */ + path: string; + ref: string; + /** The manifest with its artifacts pinned — the module as the mesh should hold it. */ + manifest: unknown; +} + +/** An artifact a module-version produced, as the builder published it. */ +export interface Made { + name: string; + kind: string; + /** How anything else names it — for an image, a registry reference pinned to a digest. */ + reference: string; +} + +/** + * A module whose artifacts were built against something that is no longer current. + * + * **An edge is an artifact reference, not a module name.** The builder can see exactly what it + * built on top of — a pinned image — and cannot see which module produced it: that is a fact about + * the graph, and the graph is here. So the reference is what is stored, and resolving it to a + * module-version is a join against what each version says it made. An edge to an artifact no + * module here produced is kept rather than dropped; it resolves by itself the day that module is + * registered, which is the ordinary case while a mesh is still being filled in. + */ + +export interface Stale { + module: string; + commit: string; + /** What moved underneath it, and to where. */ + because: { module: string; builtAgainst: string; nowAt: string }[]; +} + +const DDL = ` +CREATE TABLE IF NOT EXISTS module_version ( + module text NOT NULL, + commit_sha text NOT NULL, + repository text NOT NULL DEFAULT '', + path text NOT NULL DEFAULT '', + ref text NOT NULL DEFAULT '', + manifest jsonb NOT NULL DEFAULT '{}'::jsonb, + registered timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (module, commit_sha) +); + +-- What the catalogue considers this module to be now. Separate from module_version because a +-- module has many versions and exactly one current one, and "current" is a decision rather than +-- a consequence of being newest. +CREATE TABLE IF NOT EXISTS module_current ( + module text PRIMARY KEY, + commit_sha text NOT NULL, + moved timestamptz NOT NULL DEFAULT now() +); + +-- What a module-version published. This is what makes a build edge resolvable: an edge names an +-- artifact, and this says which version put that artifact there. +CREATE TABLE IF NOT EXISTS module_artifact ( + module text NOT NULL, + commit_sha text NOT NULL, + name text NOT NULL, + kind text NOT NULL DEFAULT '', + reference text NOT NULL, + PRIMARY KEY (module, commit_sha, name) +); + +CREATE INDEX IF NOT EXISTS module_artifact_reference ON module_artifact (reference); + +-- What a module-version declares. Separate from the build edges above and never mixed with them: +-- a build edge is a fact about what was compiled, discovered by building it, and these are +-- intentions the module states about itself (ADR 0072). They answer different questions and go +-- stale for different reasons. +CREATE TABLE IF NOT EXISTS module_requires ( + module text NOT NULL, + commit_sha text NOT NULL, + requirement text NOT NULL, + PRIMARY KEY (module, commit_sha, requirement) +); + +CREATE TABLE IF NOT EXISTS module_provides ( + module text NOT NULL, + commit_sha text NOT NULL, + provision text NOT NULL, + scope text NOT NULL DEFAULT '', + PRIMARY KEY (module, commit_sha, provision) +); + +CREATE INDEX IF NOT EXISTS module_provides_provision ON module_provides (provision); +CREATE INDEX IF NOT EXISTS module_requires_requirement ON module_requires (requirement); + +CREATE TABLE IF NOT EXISTS built_against ( + module text NOT NULL, + commit_sha text NOT NULL, + against_reference text NOT NULL, + PRIMARY KEY (module, commit_sha, against_reference) +); +`; + +// The graph first keyed its edges on a module and a commit, which the builder does not know and +// never sent — so every build that had been built against anything was rejected, and the only +// versions that registered were the ones built against nothing. No edge was ever stored, so there +// is nothing to carry across: the old columns are dropped and the new one added. +const MIGRATE = ` +ALTER TABLE built_against ADD COLUMN IF NOT EXISTS against_reference text; +DELETE FROM built_against WHERE against_reference IS NULL; +ALTER TABLE built_against DROP COLUMN IF EXISTS against_module; +ALTER TABLE built_against DROP COLUMN IF EXISTS against_commit; +ALTER TABLE built_against ALTER COLUMN against_reference SET NOT NULL; + +-- After the column exists, and not in the schema above: on a store that still has the old shape, +-- the table is not created, so an index named in the same breath would be built on a column that +-- is not there yet. +CREATE INDEX IF NOT EXISTS built_against_target ON built_against (against_reference); +`; + +export class Graph { + private constructor(private readonly pool: PgPool) {} + + static fromEnv(env: NodeJS.ProcessEnv = process.env): Graph { + const url = env["DATABASE_URL"]; + if (!url) { + throw new Error( + "no DATABASE_URL: the catalogue holds the module graph and cannot hold it in memory, " + + "because a graph that disappears on restart is not a record of anything", + ); + } + return new Graph(new Pool({ connectionString: url })); + } + + async migrate(): Promise { + await this.pool.query(DDL); + await this.pool.query(MIGRATE); + } + + /** + * Take a module-version the builder produced, and place it in the graph. + * + * **Returns whether this is an upgrade**, which is not the same as whether a build happened. A + * rebuild producing the commit already current changes nothing, and announcing it as an upgrade + * would ripple outward forever through modules that did not change (ADR 0072). + */ + async register(version: ModuleVersion, made: Made[], against: string[]): Promise<{ upgraded: boolean; previous: string | null }> { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query( + `INSERT INTO module_version (module, commit_sha, repository, path, ref, manifest) + VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (module, commit_sha) DO UPDATE SET + repository = excluded.repository, path = excluded.path, + ref = excluded.ref, manifest = excluded.manifest`, + [version.module, version.commit, version.repository, version.path, version.ref, + JSON.stringify(version.manifest ?? {})], + ); + + // Both are replaced rather than added to: they describe this build, and a previous build of + // the same commit that saw different artifacts was wrong about one of them. + // Read from the manifest rather than taken as a separate argument: the manifest the builder + // resolved IS the module's declaration, and a second copy passed alongside it could disagree + // with it. Replaced wholesale, because a version declares exactly one set of these. + const declared = (version.manifest ?? {}) as { + requires?: unknown[]; + provides?: unknown[]; + }; + await client.query(`DELETE FROM module_requires WHERE module = $1 AND commit_sha = $2`, + [version.module, version.commit]); + for (const r of declared.requires ?? []) { + const name = typeof r === "string" ? r : String((r as { name?: unknown }).name ?? ""); + if (!name) continue; + await client.query( + `INSERT INTO module_requires (module, commit_sha, requirement) VALUES ($1,$2,$3) + ON CONFLICT DO NOTHING`, [version.module, version.commit, name]); + } + await client.query(`DELETE FROM module_provides WHERE module = $1 AND commit_sha = $2`, + [version.module, version.commit]); + for (const pv of declared.provides ?? []) { + const name = typeof pv === "string" ? pv : String((pv as { name?: unknown }).name ?? ""); + const scope = typeof pv === "string" ? "" : String((pv as { scope?: unknown }).scope ?? ""); + if (!name) continue; + await client.query( + `INSERT INTO module_provides (module, commit_sha, provision, scope) VALUES ($1,$2,$3,$4) + ON CONFLICT (module, commit_sha, provision) DO UPDATE SET scope = excluded.scope`, + [version.module, version.commit, name, scope]); + } + + await client.query(`DELETE FROM module_artifact WHERE module = $1 AND commit_sha = $2`, + [version.module, version.commit]); + for (const a of made) { + await client.query( + `INSERT INTO module_artifact (module, commit_sha, name, kind, reference) + VALUES ($1,$2,$3,$4,$5) ON CONFLICT (module, commit_sha, name) DO UPDATE SET + kind = excluded.kind, reference = excluded.reference`, + [version.module, version.commit, a.name, a.kind, a.reference]); + } + + await client.query(`DELETE FROM built_against WHERE module = $1 AND commit_sha = $2`, + [version.module, version.commit]); + for (const reference of against) { + await client.query( + `INSERT INTO built_against (module, commit_sha, against_reference) + VALUES ($1,$2,$3) ON CONFLICT DO NOTHING`, + [version.module, version.commit, reference]); + } + + const was = await client.query( + `SELECT commit_sha FROM module_current WHERE module = $1`, [version.module]); + const previous = (was.rows[0]?.commit_sha as string | undefined) ?? null; + const upgraded = previous !== version.commit; + if (upgraded) { + await client.query( + `INSERT INTO module_current (module, commit_sha) VALUES ($1,$2) + ON CONFLICT (module) DO UPDATE SET commit_sha = excluded.commit_sha, moved = now()`, + [version.module, version.commit]); + } + await client.query("COMMIT"); + return { upgraded, previous }; + } catch (err) { + await this.pool.query("ROLLBACK").catch(() => {}); + throw err; + } finally { + client.release(); + } + } + + /** Every module the catalogue holds, with the commit it considers current. */ + async modules(): Promise<{ module: string; commit: string; repository: string; path: string }[]> { + const { rows } = await this.pool.query( + `SELECT c.module, c.commit_sha, v.repository, v.path + FROM module_current c + JOIN module_version v ON v.module = c.module AND v.commit_sha = c.commit_sha + ORDER BY c.module`); + return rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, + repository: r.repository as string, path: r.path as string, + })); + } + + /** One module-version in full, current unless a commit is named. */ + async show(module: string, commit?: string): Promise { + const { rows } = await this.pool.query( + commit + ? `SELECT * FROM module_version WHERE module = $1 AND commit_sha = $2` + : `SELECT v.* FROM module_version v JOIN module_current c + ON c.module = v.module AND c.commit_sha = v.commit_sha WHERE v.module = $1`, + commit ? [module, commit] : [module]); + const r = rows[0]; + if (!r) return null; + return { + module: r.module as string, commit: r.commit_sha as string, + repository: r.repository as string, path: r.path as string, + ref: r.ref as string, manifest: r.manifest, + }; + } + + /** + * What was built against this module — the modules a change to it reaches. + * + * Resolved through the artifacts each version published, because that is what an edge names. + */ + async dependents(module: string): Promise<{ module: string; commit: string; againstCommit: string }[]> { + // Distinct, because one artifact may have been published by more than one version of the + // module that made it — several commits producing a byte-identical image — and a dependent + // would otherwise be listed once per such commit, as though it were several dependents. + const { rows } = await this.pool.query( + `SELECT DISTINCT b.module, b.commit_sha, c2.commit_sha AS against_commit + FROM built_against b + JOIN module_current c ON c.module = b.module AND c.commit_sha = b.commit_sha + JOIN module_artifact a ON a.reference = b.against_reference + JOIN module_current c2 ON c2.module = a.module + WHERE a.module = $1 + ORDER BY b.module`, [module]); + return rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, + againstCommit: r.against_commit as string, + })); + } + + /** + * What must be rebuilt, and why. + * + * **The rule is a condition, not an order** (ADR 0072): a module is stale when anything it was + * built against is no longer current. Asking this repeatedly is what produces the right order — + * a module whose own dependencies are still stale simply stays stale until they are not, so a + * chain and a diamond need no special handling and nothing has to know the shape in advance. + */ + async stale(): Promise { + // **Compared by artifact, not by commit.** A commit moving is not the same as the thing it + // produces moving: a change to a comment in a build recipe is a new commit and a byte-identical + // image, and calling everything built on it stale would mean rebuilding the whole mesh to + // arrive back exactly where it started. What makes a module stale is that the artifact it was + // built against is no longer one the current version publishes. + // + // The producing version is picked as the earliest that published this artifact, because when + // several commits produce the identical image, the first one is where it actually came from. + const { rows } = await this.pool.query( + `SELECT b.module, b.commit_sha, p.module AS against_module, p.commit_sha AS against_commit, + c2.commit_sha AS now_at + FROM built_against b + JOIN module_current c1 ON c1.module = b.module AND c1.commit_sha = b.commit_sha + JOIN LATERAL ( + SELECT a.module, a.commit_sha + FROM module_artifact a + JOIN module_version v ON v.module = a.module AND v.commit_sha = a.commit_sha + WHERE a.reference = b.against_reference + ORDER BY v.registered + LIMIT 1 + ) p ON true + JOIN module_current c2 ON c2.module = p.module + WHERE NOT EXISTS ( + SELECT 1 FROM module_artifact now + WHERE now.module = c2.module AND now.commit_sha = c2.commit_sha + AND now.reference = b.against_reference) + ORDER BY b.module`); + const by = new Map(); + for (const r of rows) { + const key = `${r.module}@${r.commit_sha}`; + const entry = by.get(key) ?? { module: r.module as string, commit: r.commit_sha as string, because: [] }; + entry.because.push({ + module: r.against_module as string, + builtAgainst: r.against_commit as string, + nowAt: r.now_at as string, + }); + by.set(key, entry); + } + return [...by.values()]; + } + + /** + * Modules that are stale and whose own dependencies are all current — the ones that can be built + * now. **This is the whole of ordering.** Anything not in this set is waiting for something else, + * and will appear here once that thing is registered. + */ + async buildable(): Promise { + const stale = await this.stale(); + const waiting = new Set(stale.map((s) => s.module)); + return stale.filter((s) => !s.because.some((b) => waiting.has(b.module))); + } + + /** + * What provides a given provision, and what needs it. + * + * **Answered from the graph rather than by scanning manifests**, which is how it is answered + * today — a sweep over every module's declaration, which only whoever holds every declaration + * can do and which is wrong the moment one of them changes. + */ + async whoProvides(provision: string): Promise<{ + provides: { module: string; commit: string; scope: string }[]; + requires: { module: string; commit: string }[]; + }> { + const provides = await this.pool.query( + `SELECT p.module, p.commit_sha, p.scope + FROM module_provides p + JOIN module_current c ON c.module = p.module AND c.commit_sha = p.commit_sha + WHERE p.provision = $1 ORDER BY p.module`, [provision]); + const requires = await this.pool.query( + `SELECT r.module, r.commit_sha + FROM module_requires r + JOIN module_current c ON c.module = r.module AND c.commit_sha = r.commit_sha + WHERE r.requirement = $1 ORDER BY r.module`, [provision]); + return { + provides: provides.rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string, scope: r.scope as string })), + requires: requires.rows.map((r) => ({ + module: r.module as string, commit: r.commit_sha as string })), + }; + } + + async close(): Promise { + await this.pool.end(); + } +} diff --git a/modules/mesh-catalog/tools/index.ts b/modules/mesh-catalog/tools/index.ts new file mode 100644 index 0000000..0c4efcc --- /dev/null +++ b/modules/mesh-catalog/tools/index.ts @@ -0,0 +1,78 @@ +// mesh-catalog's tools — the module graph's query surface (novox/hq ADR 0070). +// +// These are the questions the graph exists to answer, and none of them can be answered anywhere +// else today: what does this mesh know how to run, what is this module made of, what does a change +// to this reach, and what is waiting to be rebuilt. + +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; +import { Graph } from "../store.js"; + +export function getCatalogueTools(graph: Graph): ToolDefinition[] { + return [ + { + name: "catalog_modules", + description: + "Every module this mesh holds, with the commit the catalogue considers current and where it came from.", + input: {}, + run: async () => ({ modules: await graph.modules() }), + }, + { + name: "catalog_module", + description: + "One module in full — where it came from, what it requires and provides, and what it is made of. The current version unless a commit is named.", + input: { + module: { type: "string", description: "the module's name" }, + commit: { type: "string", description: "a particular version (optional)" }, + }, + run: async (args) => { + const module = String(args.module ?? ""); + if (!module) return { error: "name a module" }; + const found = await graph.show(module, args.commit ? String(args.commit) : undefined); + return found ? { module: found } : { error: `the catalogue holds no ${module}` }; + }, + }, + { + name: "catalog_provides", + description: + "What provides a provision, and what needs it. Answered from what modules declare about themselves, so it does not require holding every manifest at once.", + input: { provision: { type: "string", description: "the provision, e.g. postgres-database" } }, + run: async (args) => { + const provision = String(args.provision ?? ""); + if (!provision) return { error: "name a provision" }; + return await graph.whoProvides(provision); + }, + }, + { + name: "catalog_dependents", + description: + "What was built against this module — the modules a change to it reaches. Derived from builds, not from a declared list, so it cannot drift from what the code actually uses.", + input: { module: { type: "string", description: "the module that would change" } }, + run: async (args) => { + const module = String(args.module ?? ""); + if (!module) return { error: "name a module" }; + return { module, dependents: await graph.dependents(module) }; + }, + }, + { + name: "catalog_stale", + description: + "What must be rebuilt and why — every module built against something that has since moved. `buildable` is the subset waiting on nothing that is itself stale, which is the set that can be built right now.", + input: {}, + run: async () => ({ + stale: await graph.stale(), + buildable: await graph.buildable(), + }), + }, + ]; +} + +// Opened from the environment when the runtime asks for the module's tools. When it cannot be — +// no DATABASE_URL — the module contributes no tools rather than taking the whole tool runtime down +// with it, which is the postgres precedent. +registerModuleTools("mesh-catalog", (env) => { + try { + return getCatalogueTools(Graph.fromEnv(env)); + } catch { + return []; + } +}); diff --git a/modules/mesh-catalog/tsconfig.json b/modules/mesh-catalog/tsconfig.json new file mode 100644 index 0000000..2528f05 --- /dev/null +++ b/modules/mesh-catalog/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": [ + "pg.d.ts", + "store.ts", + "index.ts", + "tools/index.ts" + ] +} diff --git a/modules/postgres/Dockerfile b/modules/postgres/Dockerfile new file mode 100644 index 0000000..aa74659 --- /dev/null +++ b/modules/postgres/Dockerfile @@ -0,0 +1,36 @@ +# postgres's runtime: the tool runtime, carrying this module's compiled provisioner, tools and +# event consumer. +# +# **Built from this module's own directory and nothing else.** The sdk is in the base image, so +# nothing is copied out of a neighbouring checkout — which is what lets the mesh build this from a +# repository and a path (novox/hq ADR 0069) rather than only on a workstation that happens to have +# the siblings. +# +# The base is named by ARG so it can be pinned to a digest the mesh's registry assigned. A tag would +# make this runtime's contents depend on what somebody last pushed under that name. +ARG RUNTIME_BASE=127.0.0.1:5000/mesh-tools/runtime@sha256:44b5d8bc30107fdc3bffdaebc1ca2de97615053853f826dfccce253a01a163fd + +FROM ${RUNTIME_BASE} AS build +# Compiled under /app/modules so `@novox/mesh-sdk` resolves upward into the base's own +# node_modules — the module is compiled against exactly the sdk it will run against. +WORKDIR /app/modules/postgres +COPY . . +# The compiler is invoked by its real path rather than through node_modules/.bin, whose entries are +# symlinks to a launcher that requires its library relatively — resolved away when the base image +# was assembled. +RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts provisioner/index.ts tools/index.ts \ + --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist + +FROM ${RUNTIME_BASE} +# **This module talks to its database through psql, so psql has to be here.** The client is how +# postgres's provisioner runs DDL — it does not carry a driver — and the runtime base holds only +# what every module needs. +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /app/modules/postgres/dist /app/modules/postgres/dist +# What a tool host should load from this module: its event consumer and its tools, which are +# separate entrypoints because they are loaded by different things. The provisioner is the third, +# and is not listed here — the declaration names it in the container's `args`, because it is what +# this module's own container runs. One image, because they are one module and share a client. +ENV MESH_TOOL_MODULES=/app/modules/postgres/dist/index.js,/app/modules/postgres/dist/tools/index.js diff --git a/modules/postgres/module.json b/modules/postgres/module.json index e6d9089..a161300 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -102,7 +102,6 @@ "id": "runtime", "type": "container", "name": "mesh-postgres", - "image": "mesh-runtime-postgres@sha256:0000000000000000000000000000000000000000000000000000000000000000", "network": "postgres", "volumes": [ "/var/lib/mesh/postgres/broker:/run/secrets/broker:ro", @@ -114,7 +113,21 @@ "MESH_PROVISION_PASSWORD_FILE": "/run/secrets/superuser", "MESH_BROKER_FILE": "/run/secrets/broker", "MESH_RECEIVES": "/var/lib/postgres/grants/mesh.json" - } + }, + "artifact": "runtime", + "args": [ + "run", + "/app/modules/postgres/dist/provisioner/index.js" + ] } - ] + ], + "build": { + "artifacts": [ + { + "name": "runtime", + "kind": "image", + "from": "Dockerfile" + } + ] + } }