Merge pull request 'The catalogue holds the module graph, and the modules that make it build themselves' (#21) from feat/the-catalogue-module into main
This commit was merged in pull request #21.
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
# amqp-ping's runtime: the tool runtime, carrying this module's compiled code.
|
||||
#
|
||||
# **Built from this module's own directory and nothing else.** The sdk and the tool runtime are not
|
||||
# copied out of neighbouring checkouts — they are in the base image, which is published like any
|
||||
# other artifact. That is what makes this buildable by the mesh from a repository and a path
|
||||
# (novox/hq ADR 0069) rather than only on a workstation that happens to have the siblings.
|
||||
#
|
||||
# The base is named by ARG so it can be pinned to a digest the mesh's registry assigned. A tag here
|
||||
# would make the runtime's contents depend on what somebody last pushed under that name.
|
||||
ARG RUNTIME_BASE=127.0.0.1:5000/mesh-tools/runtime@sha256:44b5d8bc30107fdc3bffdaebc1ca2de97615053853f826dfccce253a01a163fd
|
||||
|
||||
FROM ${RUNTIME_BASE} AS build
|
||||
# Compiled under /app/modules, so resolving `@novox/mesh-sdk` walks up to the base's own
|
||||
# node_modules — the module is compiled against exactly the sdk it will run against.
|
||||
WORKDIR /app/modules/amqp-ping
|
||||
COPY . .
|
||||
# The compiler is invoked by its real path, not through node_modules/.bin. Those are symlinks to
|
||||
# a launcher that requires its library relatively, and the base image resolves them when copying —
|
||||
# leaving a launcher whose relative require no longer points at anything.
|
||||
RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts \
|
||||
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
|
||||
|
||||
FROM ${RUNTIME_BASE}
|
||||
COPY --from=build /app/modules/amqp-ping/dist /app/modules/amqp-ping/dist
|
||||
# Declared rather than derived from which files happen to exist: the module knows what it serves.
|
||||
ENV MESH_TOOL_MODULES=/app/modules/amqp-ping/dist/index.js
|
||||
@@ -47,18 +47,23 @@
|
||||
"id": "runtime",
|
||||
"type": "container",
|
||||
"name": "amqp-ping",
|
||||
"image": "mesh-runtime-amqp-ping@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"network": "amqp-ping",
|
||||
"env-file": [
|
||||
"/var/lib/amqp-ping/amqp.env"
|
||||
],
|
||||
"args": [
|
||||
"run",
|
||||
"/app/modules/amqp-ping/dist/index.js"
|
||||
],
|
||||
"restart-on": [
|
||||
"amqp-env"
|
||||
]
|
||||
],
|
||||
"artifact": "runtime"
|
||||
}
|
||||
]
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
"requires": [
|
||||
"artifact-store"
|
||||
],
|
||||
"emits": [
|
||||
"module.builder.built"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "/var/lib/mesh/builder/broker"
|
||||
},
|
||||
@@ -34,7 +37,7 @@
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh/builder/builder.env",
|
||||
"mode": "0600",
|
||||
"content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_REGISTRY=127.0.0.1:${bound:artifact-store:port}\nMESH_WORKSPACE=/workspace\n"
|
||||
"content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=127.0.0.1:${bound:artifact-store:port}\nMESH_WORKSPACE=/workspace\n"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
|
||||
@@ -48,7 +48,6 @@
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
"name": "hello-web",
|
||||
"image": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b",
|
||||
"network": "hello-web",
|
||||
"ports": [
|
||||
"8080:8080"
|
||||
@@ -60,7 +59,17 @@
|
||||
"sh",
|
||||
"-c",
|
||||
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done"
|
||||
]
|
||||
],
|
||||
"artifact": "server"
|
||||
}
|
||||
]
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "server",
|
||||
"kind": "upstream",
|
||||
"from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# mesh-catalog's runtime: the tool runtime, carrying the catalogue's compiled graph, its consumer
|
||||
# of what the builder announces, and its tools.
|
||||
#
|
||||
# **Built from this module's own directory and nothing else.** The sdk is in the base image, so
|
||||
# nothing is copied out of a neighbouring checkout — which is what lets the mesh build this from a
|
||||
# repository and a path (novox/hq ADR 0069) rather than only on a workstation with the siblings.
|
||||
#
|
||||
# The base is named by ARG so it can be pinned to a digest the mesh's registry assigned. A tag would
|
||||
# make this runtime's contents depend on what somebody last pushed under that name.
|
||||
ARG RUNTIME_BASE=127.0.0.1:5000/mesh-tools/runtime@sha256:44b5d8bc30107fdc3bffdaebc1ca2de97615053853f826dfccce253a01a163fd
|
||||
|
||||
FROM ${RUNTIME_BASE} AS build
|
||||
# Compiled under /app/modules so `@novox/mesh-sdk` resolves upward into the base's own
|
||||
# node_modules — the module is compiled against exactly the sdk it will run against.
|
||||
WORKDIR /app/modules/mesh-catalog
|
||||
COPY . .
|
||||
# The compiler is invoked by its real path rather than through node_modules/.bin, whose entries are
|
||||
# symlinks to a launcher that requires its library relatively — resolved away when the base image
|
||||
# was assembled.
|
||||
RUN node /app/node_modules/typescript/bin/tsc pg.d.ts store.ts index.ts tools/index.ts \
|
||||
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
|
||||
|
||||
# **A module may need something the base image does not carry.** The base holds what every module
|
||||
# needs — the sdk, the broker client — and a postgres driver is not that: the one other module that
|
||||
# reaches a database shells out to psql instead. So the catalogue brings its own.
|
||||
#
|
||||
# Installed into an empty directory rather than into the module's, because the module's package.json
|
||||
# also names `@novox/mesh-sdk`, which is not on any registry — it is in the base image. Asking npm to
|
||||
# resolve this module's dependencies would therefore fail on the one it already has.
|
||||
RUN mkdir -p /deps && cd /deps && \
|
||||
npm install --omit=dev --no-audit --no-fund --no-package-lock pg@8
|
||||
|
||||
FROM ${RUNTIME_BASE}
|
||||
COPY --from=build /app/modules/mesh-catalog/dist /app/modules/mesh-catalog/dist
|
||||
# Beside the compiled code, so `pg` resolves from it while `@novox/mesh-sdk` keeps walking up to the
|
||||
# base image's own node_modules — the module gets its extra dependency without shadowing the sdk it
|
||||
# was compiled against.
|
||||
COPY --from=build /deps/node_modules /app/modules/mesh-catalog/node_modules
|
||||
# Both entrypoints, loaded in serve mode.
|
||||
#
|
||||
# **A consumer cannot be started with `run`.** That mode imports an entrypoint without binding a
|
||||
# broker — it is for a step that does its work offline and exits — and the catalogue's whole job is
|
||||
# to listen for what the builder announces. Serve binds the broker first, then imports these, so
|
||||
# `on()` has something to subscribe to.
|
||||
ENV MESH_TOOL_MODULES=/app/modules/mesh-catalog/dist/index.js,/app/modules/mesh-catalog/dist/tools/index.js
|
||||
@@ -0,0 +1,79 @@
|
||||
// mesh-catalog's entrypoint — the module graph's consumer (novox/hq ADR 0070, ADR 0072).
|
||||
//
|
||||
// The builder announces what it built; this places it in the graph and announces what that means.
|
||||
// The control plane hooks the *meaning* — a module was upgraded — rather than the build output, so
|
||||
// it never has to interpret an artifact or ask this module anything.
|
||||
//
|
||||
// **Ordering is not computed here.** When a registration makes something else stale, the modules
|
||||
// whose own dependencies are all current are announced as needing a rebuild; the rest stay stale
|
||||
// and appear the next time round, once whatever they were waiting for is registered. A chain and a
|
||||
// diamond need no special handling, and nothing holds a plan.
|
||||
|
||||
import { on, emit } from "@novox/mesh-sdk/events";
|
||||
import { Graph, type Made } from "./store.js";
|
||||
|
||||
const graph = Graph.fromEnv();
|
||||
|
||||
// Before subscribing, and idempotent. The runtime is restarted until its store is reachable, which
|
||||
// is the same arrangement model-usage uses: a schema step that had to reach the provider over the
|
||||
// overlay would block the very apply that brings the overlay up.
|
||||
await graph.migrate();
|
||||
|
||||
/** What the builder says when it has built something. */
|
||||
interface Built {
|
||||
module?: string;
|
||||
commit?: string;
|
||||
repository?: string;
|
||||
path?: string;
|
||||
ref?: string;
|
||||
manifest?: unknown;
|
||||
/** What this build published, each pinned as anything else would name it. */
|
||||
made?: Made[];
|
||||
/**
|
||||
* Every artifact this was built on top of, so the edge is derived rather than declared.
|
||||
*
|
||||
* References, not module names: the builder sees a pinned image and cannot see which module
|
||||
* produced it. Turning that into an edge between module-versions is this module's job.
|
||||
*/
|
||||
against?: string[];
|
||||
}
|
||||
|
||||
await on("module.builder.built", async (event) => {
|
||||
const body = event.body as Built;
|
||||
if (!body.module || !body.commit) {
|
||||
// Said rather than dropped: a build that announced itself without saying what it built is a
|
||||
// fault in the builder, and a silent discard here would make it look like a missing event.
|
||||
console.error("mesh-catalog: a build event named no module or no commit; ignored", body);
|
||||
return;
|
||||
}
|
||||
|
||||
const { upgraded, previous } = await graph.register({
|
||||
module: body.module,
|
||||
commit: body.commit,
|
||||
repository: body.repository ?? "",
|
||||
path: body.path ?? "",
|
||||
ref: body.ref ?? "",
|
||||
manifest: body.manifest ?? {},
|
||||
}, body.made ?? [], body.against ?? []);
|
||||
|
||||
await emit("module.mesh-catalog.registered", {
|
||||
module: body.module, commit: body.commit, upgraded,
|
||||
});
|
||||
|
||||
// **A rebuild that changed nothing is not an upgrade.** Announcing it would ripple outward
|
||||
// through modules that did not change, forever (ADR 0072).
|
||||
if (!upgraded) return;
|
||||
|
||||
await emit("module.mesh-catalog.upgraded", {
|
||||
module: body.module, commit: body.commit, previous,
|
||||
});
|
||||
|
||||
// What can be built now — stale, and waiting on nothing that is itself stale.
|
||||
for (const next of await graph.buildable()) {
|
||||
await emit("module.mesh-catalog.rebuild-needed", {
|
||||
module: next.module,
|
||||
builtAt: next.commit,
|
||||
because: next.because,
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"module": "mesh-catalog",
|
||||
"slug": "catalog",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "the-catalogue",
|
||||
"scope": "mesh"
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"postgres-database"
|
||||
],
|
||||
"contributes": {
|
||||
"postgres-database": {
|
||||
"name": "mesh_catalog"
|
||||
}
|
||||
},
|
||||
"binds": {
|
||||
"postgres-database": "/var/lib/mesh-catalog/database.json"
|
||||
},
|
||||
"secrets": {
|
||||
"postgres-database": "/var/lib/mesh-catalog/database.secret"
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "/var/lib/mesh/mesh-catalog/broker"
|
||||
},
|
||||
"consumes": [
|
||||
"module.builder.built"
|
||||
],
|
||||
"emits": [
|
||||
"module.mesh-catalog.registered",
|
||||
"module.mesh-catalog.upgraded",
|
||||
"module.mesh-catalog.rebuild-needed"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "mesh-state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh/mesh-catalog",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-catalog",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "db-env",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-catalog/db.env",
|
||||
"mode": "0600",
|
||||
"content": "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n"
|
||||
},
|
||||
{
|
||||
"id": "runtime",
|
||||
"type": "container",
|
||||
"name": "mesh-catalog",
|
||||
"network": "host",
|
||||
"volumes": [
|
||||
"/var/lib/mesh/mesh-catalog/broker:/run/secrets/broker:ro",
|
||||
"/var/lib/mesh-catalog:/run/state"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "/run/secrets/broker"
|
||||
},
|
||||
"env-file": [
|
||||
"/var/lib/mesh-catalog/db.env"
|
||||
],
|
||||
"artifact": "runtime",
|
||||
"restart-on": [
|
||||
"db-env"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@novox/module-mesh-catalog",
|
||||
"version": "0.1.0",
|
||||
"description": "mesh-catalog — the module graph (novox/hq ADR 0070, 0072): links module-versions to each other, registers what the builder announces, and says what a change reaches and what can be rebuilt now. Knows nothing about nodes.",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.0",
|
||||
"pg": "^8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.6.0"
|
||||
}
|
||||
}
|
||||
Vendored
+22
@@ -0,0 +1,22 @@
|
||||
// Ambient types for `pg` (node-postgres), which ships its types only via the separate `@types/pg`
|
||||
// package. Rather than pull that in at tsc time, this declares the exact slice model-usage uses —
|
||||
// the same precedent anthropic-manager sets for `tweetnacl-sealedbox-js` (a local ambient .d.ts,
|
||||
// listed in tsconfig `include`, default-imported). The real `pg` is installed into the module's
|
||||
// runtime image (package.json `dependencies`; novox/hq ADR 0052), so this types the code without
|
||||
// deciding what runs.
|
||||
declare module "pg" {
|
||||
/** One checked-out connection. Needed because registering a module-version and its edges is one
|
||||
* act: a half-written registration is a graph that lies about what something was built against. */
|
||||
export class PoolClient {
|
||||
query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>;
|
||||
release(): void;
|
||||
}
|
||||
/** A lazily-connecting connection pool. Only the connection string, one query form, checking out
|
||||
* a connection, and end() are used here. */
|
||||
export class Pool {
|
||||
constructor(config?: { connectionString?: string });
|
||||
query(text: string, params?: unknown[]): Promise<{ rows: any[]; rowCount: number }>;
|
||||
connect(): Promise<PoolClient>;
|
||||
end(): Promise<void>;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
// The module graph (novox/hq ADR 0070, ADR 0072).
|
||||
//
|
||||
// **This graph links module-versions to each other and knows nothing about nodes.** Which machine
|
||||
// runs what is the control plane's graph, and the two meet only when somebody installs something.
|
||||
// Keeping them apart is what lets the control plane carry on composing declarations while this is
|
||||
// down: it holds what it needs, and asks nothing here.
|
||||
//
|
||||
// A module-version is identified by its commit, not by a version string. A version is what somebody
|
||||
// calls a release; a commit is what was actually built (ADR 0009).
|
||||
|
||||
import pg from "pg";
|
||||
|
||||
// `pg` is CommonJS: its default export is the module object, so Pool is a property of it. This is
|
||||
// the form that resolves under Node's ESM loader — the same shape model-usage uses.
|
||||
const { Pool } = pg;
|
||||
type PgPool = InstanceType<typeof Pool>;
|
||||
|
||||
/** One module-version, as the builder delivered it. */
|
||||
export interface ModuleVersion {
|
||||
module: string;
|
||||
commit: string;
|
||||
repository: string;
|
||||
/** The module's directory inside that repository (novox/hq ADR 0069). Empty is the root. */
|
||||
path: string;
|
||||
ref: string;
|
||||
/** The manifest with its artifacts pinned — the module as the mesh should hold it. */
|
||||
manifest: unknown;
|
||||
}
|
||||
|
||||
/** An artifact a module-version produced, as the builder published it. */
|
||||
export interface Made {
|
||||
name: string;
|
||||
kind: string;
|
||||
/** How anything else names it — for an image, a registry reference pinned to a digest. */
|
||||
reference: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A module whose artifacts were built against something that is no longer current.
|
||||
*
|
||||
* **An edge is an artifact reference, not a module name.** The builder can see exactly what it
|
||||
* built on top of — a pinned image — and cannot see which module produced it: that is a fact about
|
||||
* the graph, and the graph is here. So the reference is what is stored, and resolving it to a
|
||||
* module-version is a join against what each version says it made. An edge to an artifact no
|
||||
* module here produced is kept rather than dropped; it resolves by itself the day that module is
|
||||
* registered, which is the ordinary case while a mesh is still being filled in.
|
||||
*/
|
||||
|
||||
export interface Stale {
|
||||
module: string;
|
||||
commit: string;
|
||||
/** What moved underneath it, and to where. */
|
||||
because: { module: string; builtAgainst: string; nowAt: string }[];
|
||||
}
|
||||
|
||||
const DDL = `
|
||||
CREATE TABLE IF NOT EXISTS module_version (
|
||||
module text NOT NULL,
|
||||
commit_sha text NOT NULL,
|
||||
repository text NOT NULL DEFAULT '',
|
||||
path text NOT NULL DEFAULT '',
|
||||
ref text NOT NULL DEFAULT '',
|
||||
manifest jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
registered timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (module, commit_sha)
|
||||
);
|
||||
|
||||
-- What the catalogue considers this module to be now. Separate from module_version because a
|
||||
-- module has many versions and exactly one current one, and "current" is a decision rather than
|
||||
-- a consequence of being newest.
|
||||
CREATE TABLE IF NOT EXISTS module_current (
|
||||
module text PRIMARY KEY,
|
||||
commit_sha text NOT NULL,
|
||||
moved timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- What a module-version published. This is what makes a build edge resolvable: an edge names an
|
||||
-- artifact, and this says which version put that artifact there.
|
||||
CREATE TABLE IF NOT EXISTS module_artifact (
|
||||
module text NOT NULL,
|
||||
commit_sha text NOT NULL,
|
||||
name text NOT NULL,
|
||||
kind text NOT NULL DEFAULT '',
|
||||
reference text NOT NULL,
|
||||
PRIMARY KEY (module, commit_sha, name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS module_artifact_reference ON module_artifact (reference);
|
||||
|
||||
-- What a module-version declares. Separate from the build edges above and never mixed with them:
|
||||
-- a build edge is a fact about what was compiled, discovered by building it, and these are
|
||||
-- intentions the module states about itself (ADR 0072). They answer different questions and go
|
||||
-- stale for different reasons.
|
||||
CREATE TABLE IF NOT EXISTS module_requires (
|
||||
module text NOT NULL,
|
||||
commit_sha text NOT NULL,
|
||||
requirement text NOT NULL,
|
||||
PRIMARY KEY (module, commit_sha, requirement)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS module_provides (
|
||||
module text NOT NULL,
|
||||
commit_sha text NOT NULL,
|
||||
provision text NOT NULL,
|
||||
scope text NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (module, commit_sha, provision)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS module_provides_provision ON module_provides (provision);
|
||||
CREATE INDEX IF NOT EXISTS module_requires_requirement ON module_requires (requirement);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS built_against (
|
||||
module text NOT NULL,
|
||||
commit_sha text NOT NULL,
|
||||
against_reference text NOT NULL,
|
||||
PRIMARY KEY (module, commit_sha, against_reference)
|
||||
);
|
||||
`;
|
||||
|
||||
// The graph first keyed its edges on a module and a commit, which the builder does not know and
|
||||
// never sent — so every build that had been built against anything was rejected, and the only
|
||||
// versions that registered were the ones built against nothing. No edge was ever stored, so there
|
||||
// is nothing to carry across: the old columns are dropped and the new one added.
|
||||
const MIGRATE = `
|
||||
ALTER TABLE built_against ADD COLUMN IF NOT EXISTS against_reference text;
|
||||
DELETE FROM built_against WHERE against_reference IS NULL;
|
||||
ALTER TABLE built_against DROP COLUMN IF EXISTS against_module;
|
||||
ALTER TABLE built_against DROP COLUMN IF EXISTS against_commit;
|
||||
ALTER TABLE built_against ALTER COLUMN against_reference SET NOT NULL;
|
||||
|
||||
-- After the column exists, and not in the schema above: on a store that still has the old shape,
|
||||
-- the table is not created, so an index named in the same breath would be built on a column that
|
||||
-- is not there yet.
|
||||
CREATE INDEX IF NOT EXISTS built_against_target ON built_against (against_reference);
|
||||
`;
|
||||
|
||||
export class Graph {
|
||||
private constructor(private readonly pool: PgPool) {}
|
||||
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): Graph {
|
||||
const url = env["DATABASE_URL"];
|
||||
if (!url) {
|
||||
throw new Error(
|
||||
"no DATABASE_URL: the catalogue holds the module graph and cannot hold it in memory, " +
|
||||
"because a graph that disappears on restart is not a record of anything",
|
||||
);
|
||||
}
|
||||
return new Graph(new Pool({ connectionString: url }));
|
||||
}
|
||||
|
||||
async migrate(): Promise<void> {
|
||||
await this.pool.query(DDL);
|
||||
await this.pool.query(MIGRATE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a module-version the builder produced, and place it in the graph.
|
||||
*
|
||||
* **Returns whether this is an upgrade**, which is not the same as whether a build happened. A
|
||||
* rebuild producing the commit already current changes nothing, and announcing it as an upgrade
|
||||
* would ripple outward forever through modules that did not change (ADR 0072).
|
||||
*/
|
||||
async register(version: ModuleVersion, made: Made[], against: string[]): Promise<{ upgraded: boolean; previous: string | null }> {
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query(
|
||||
`INSERT INTO module_version (module, commit_sha, repository, path, ref, manifest)
|
||||
VALUES ($1,$2,$3,$4,$5,$6)
|
||||
ON CONFLICT (module, commit_sha) DO UPDATE SET
|
||||
repository = excluded.repository, path = excluded.path,
|
||||
ref = excluded.ref, manifest = excluded.manifest`,
|
||||
[version.module, version.commit, version.repository, version.path, version.ref,
|
||||
JSON.stringify(version.manifest ?? {})],
|
||||
);
|
||||
|
||||
// Both are replaced rather than added to: they describe this build, and a previous build of
|
||||
// the same commit that saw different artifacts was wrong about one of them.
|
||||
// Read from the manifest rather than taken as a separate argument: the manifest the builder
|
||||
// resolved IS the module's declaration, and a second copy passed alongside it could disagree
|
||||
// with it. Replaced wholesale, because a version declares exactly one set of these.
|
||||
const declared = (version.manifest ?? {}) as {
|
||||
requires?: unknown[];
|
||||
provides?: unknown[];
|
||||
};
|
||||
await client.query(`DELETE FROM module_requires WHERE module = $1 AND commit_sha = $2`,
|
||||
[version.module, version.commit]);
|
||||
for (const r of declared.requires ?? []) {
|
||||
const name = typeof r === "string" ? r : String((r as { name?: unknown }).name ?? "");
|
||||
if (!name) continue;
|
||||
await client.query(
|
||||
`INSERT INTO module_requires (module, commit_sha, requirement) VALUES ($1,$2,$3)
|
||||
ON CONFLICT DO NOTHING`, [version.module, version.commit, name]);
|
||||
}
|
||||
await client.query(`DELETE FROM module_provides WHERE module = $1 AND commit_sha = $2`,
|
||||
[version.module, version.commit]);
|
||||
for (const pv of declared.provides ?? []) {
|
||||
const name = typeof pv === "string" ? pv : String((pv as { name?: unknown }).name ?? "");
|
||||
const scope = typeof pv === "string" ? "" : String((pv as { scope?: unknown }).scope ?? "");
|
||||
if (!name) continue;
|
||||
await client.query(
|
||||
`INSERT INTO module_provides (module, commit_sha, provision, scope) VALUES ($1,$2,$3,$4)
|
||||
ON CONFLICT (module, commit_sha, provision) DO UPDATE SET scope = excluded.scope`,
|
||||
[version.module, version.commit, name, scope]);
|
||||
}
|
||||
|
||||
await client.query(`DELETE FROM module_artifact WHERE module = $1 AND commit_sha = $2`,
|
||||
[version.module, version.commit]);
|
||||
for (const a of made) {
|
||||
await client.query(
|
||||
`INSERT INTO module_artifact (module, commit_sha, name, kind, reference)
|
||||
VALUES ($1,$2,$3,$4,$5) ON CONFLICT (module, commit_sha, name) DO UPDATE SET
|
||||
kind = excluded.kind, reference = excluded.reference`,
|
||||
[version.module, version.commit, a.name, a.kind, a.reference]);
|
||||
}
|
||||
|
||||
await client.query(`DELETE FROM built_against WHERE module = $1 AND commit_sha = $2`,
|
||||
[version.module, version.commit]);
|
||||
for (const reference of against) {
|
||||
await client.query(
|
||||
`INSERT INTO built_against (module, commit_sha, against_reference)
|
||||
VALUES ($1,$2,$3) ON CONFLICT DO NOTHING`,
|
||||
[version.module, version.commit, reference]);
|
||||
}
|
||||
|
||||
const was = await client.query(
|
||||
`SELECT commit_sha FROM module_current WHERE module = $1`, [version.module]);
|
||||
const previous = (was.rows[0]?.commit_sha as string | undefined) ?? null;
|
||||
const upgraded = previous !== version.commit;
|
||||
if (upgraded) {
|
||||
await client.query(
|
||||
`INSERT INTO module_current (module, commit_sha) VALUES ($1,$2)
|
||||
ON CONFLICT (module) DO UPDATE SET commit_sha = excluded.commit_sha, moved = now()`,
|
||||
[version.module, version.commit]);
|
||||
}
|
||||
await client.query("COMMIT");
|
||||
return { upgraded, previous };
|
||||
} catch (err) {
|
||||
await this.pool.query("ROLLBACK").catch(() => {});
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Every module the catalogue holds, with the commit it considers current. */
|
||||
async modules(): Promise<{ module: string; commit: string; repository: string; path: string }[]> {
|
||||
const { rows } = await this.pool.query(
|
||||
`SELECT c.module, c.commit_sha, v.repository, v.path
|
||||
FROM module_current c
|
||||
JOIN module_version v ON v.module = c.module AND v.commit_sha = c.commit_sha
|
||||
ORDER BY c.module`);
|
||||
return rows.map((r) => ({
|
||||
module: r.module as string, commit: r.commit_sha as string,
|
||||
repository: r.repository as string, path: r.path as string,
|
||||
}));
|
||||
}
|
||||
|
||||
/** One module-version in full, current unless a commit is named. */
|
||||
async show(module: string, commit?: string): Promise<ModuleVersion | null> {
|
||||
const { rows } = await this.pool.query(
|
||||
commit
|
||||
? `SELECT * FROM module_version WHERE module = $1 AND commit_sha = $2`
|
||||
: `SELECT v.* FROM module_version v JOIN module_current c
|
||||
ON c.module = v.module AND c.commit_sha = v.commit_sha WHERE v.module = $1`,
|
||||
commit ? [module, commit] : [module]);
|
||||
const r = rows[0];
|
||||
if (!r) return null;
|
||||
return {
|
||||
module: r.module as string, commit: r.commit_sha as string,
|
||||
repository: r.repository as string, path: r.path as string,
|
||||
ref: r.ref as string, manifest: r.manifest,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* What was built against this module — the modules a change to it reaches.
|
||||
*
|
||||
* Resolved through the artifacts each version published, because that is what an edge names.
|
||||
*/
|
||||
async dependents(module: string): Promise<{ module: string; commit: string; againstCommit: string }[]> {
|
||||
// Distinct, because one artifact may have been published by more than one version of the
|
||||
// module that made it — several commits producing a byte-identical image — and a dependent
|
||||
// would otherwise be listed once per such commit, as though it were several dependents.
|
||||
const { rows } = await this.pool.query(
|
||||
`SELECT DISTINCT b.module, b.commit_sha, c2.commit_sha AS against_commit
|
||||
FROM built_against b
|
||||
JOIN module_current c ON c.module = b.module AND c.commit_sha = b.commit_sha
|
||||
JOIN module_artifact a ON a.reference = b.against_reference
|
||||
JOIN module_current c2 ON c2.module = a.module
|
||||
WHERE a.module = $1
|
||||
ORDER BY b.module`, [module]);
|
||||
return rows.map((r) => ({
|
||||
module: r.module as string, commit: r.commit_sha as string,
|
||||
againstCommit: r.against_commit as string,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* What must be rebuilt, and why.
|
||||
*
|
||||
* **The rule is a condition, not an order** (ADR 0072): a module is stale when anything it was
|
||||
* built against is no longer current. Asking this repeatedly is what produces the right order —
|
||||
* a module whose own dependencies are still stale simply stays stale until they are not, so a
|
||||
* chain and a diamond need no special handling and nothing has to know the shape in advance.
|
||||
*/
|
||||
async stale(): Promise<Stale[]> {
|
||||
// **Compared by artifact, not by commit.** A commit moving is not the same as the thing it
|
||||
// produces moving: a change to a comment in a build recipe is a new commit and a byte-identical
|
||||
// image, and calling everything built on it stale would mean rebuilding the whole mesh to
|
||||
// arrive back exactly where it started. What makes a module stale is that the artifact it was
|
||||
// built against is no longer one the current version publishes.
|
||||
//
|
||||
// The producing version is picked as the earliest that published this artifact, because when
|
||||
// several commits produce the identical image, the first one is where it actually came from.
|
||||
const { rows } = await this.pool.query(
|
||||
`SELECT b.module, b.commit_sha, p.module AS against_module, p.commit_sha AS against_commit,
|
||||
c2.commit_sha AS now_at
|
||||
FROM built_against b
|
||||
JOIN module_current c1 ON c1.module = b.module AND c1.commit_sha = b.commit_sha
|
||||
JOIN LATERAL (
|
||||
SELECT a.module, a.commit_sha
|
||||
FROM module_artifact a
|
||||
JOIN module_version v ON v.module = a.module AND v.commit_sha = a.commit_sha
|
||||
WHERE a.reference = b.against_reference
|
||||
ORDER BY v.registered
|
||||
LIMIT 1
|
||||
) p ON true
|
||||
JOIN module_current c2 ON c2.module = p.module
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM module_artifact now
|
||||
WHERE now.module = c2.module AND now.commit_sha = c2.commit_sha
|
||||
AND now.reference = b.against_reference)
|
||||
ORDER BY b.module`);
|
||||
const by = new Map<string, Stale>();
|
||||
for (const r of rows) {
|
||||
const key = `${r.module}@${r.commit_sha}`;
|
||||
const entry = by.get(key) ?? { module: r.module as string, commit: r.commit_sha as string, because: [] };
|
||||
entry.because.push({
|
||||
module: r.against_module as string,
|
||||
builtAgainst: r.against_commit as string,
|
||||
nowAt: r.now_at as string,
|
||||
});
|
||||
by.set(key, entry);
|
||||
}
|
||||
return [...by.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Modules that are stale and whose own dependencies are all current — the ones that can be built
|
||||
* now. **This is the whole of ordering.** Anything not in this set is waiting for something else,
|
||||
* and will appear here once that thing is registered.
|
||||
*/
|
||||
async buildable(): Promise<Stale[]> {
|
||||
const stale = await this.stale();
|
||||
const waiting = new Set(stale.map((s) => s.module));
|
||||
return stale.filter((s) => !s.because.some((b) => waiting.has(b.module)));
|
||||
}
|
||||
|
||||
/**
|
||||
* What provides a given provision, and what needs it.
|
||||
*
|
||||
* **Answered from the graph rather than by scanning manifests**, which is how it is answered
|
||||
* today — a sweep over every module's declaration, which only whoever holds every declaration
|
||||
* can do and which is wrong the moment one of them changes.
|
||||
*/
|
||||
async whoProvides(provision: string): Promise<{
|
||||
provides: { module: string; commit: string; scope: string }[];
|
||||
requires: { module: string; commit: string }[];
|
||||
}> {
|
||||
const provides = await this.pool.query(
|
||||
`SELECT p.module, p.commit_sha, p.scope
|
||||
FROM module_provides p
|
||||
JOIN module_current c ON c.module = p.module AND c.commit_sha = p.commit_sha
|
||||
WHERE p.provision = $1 ORDER BY p.module`, [provision]);
|
||||
const requires = await this.pool.query(
|
||||
`SELECT r.module, r.commit_sha
|
||||
FROM module_requires r
|
||||
JOIN module_current c ON c.module = r.module AND c.commit_sha = r.commit_sha
|
||||
WHERE r.requirement = $1 ORDER BY r.module`, [provision]);
|
||||
return {
|
||||
provides: provides.rows.map((r) => ({
|
||||
module: r.module as string, commit: r.commit_sha as string, scope: r.scope as string })),
|
||||
requires: requires.rows.map((r) => ({
|
||||
module: r.module as string, commit: r.commit_sha as string })),
|
||||
};
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.pool.end();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// mesh-catalog's tools — the module graph's query surface (novox/hq ADR 0070).
|
||||
//
|
||||
// These are the questions the graph exists to answer, and none of them can be answered anywhere
|
||||
// else today: what does this mesh know how to run, what is this module made of, what does a change
|
||||
// to this reach, and what is waiting to be rebuilt.
|
||||
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { Graph } from "../store.js";
|
||||
|
||||
export function getCatalogueTools(graph: Graph): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "catalog_modules",
|
||||
description:
|
||||
"Every module this mesh holds, with the commit the catalogue considers current and where it came from.",
|
||||
input: {},
|
||||
run: async () => ({ modules: await graph.modules() }),
|
||||
},
|
||||
{
|
||||
name: "catalog_module",
|
||||
description:
|
||||
"One module in full — where it came from, what it requires and provides, and what it is made of. The current version unless a commit is named.",
|
||||
input: {
|
||||
module: { type: "string", description: "the module's name" },
|
||||
commit: { type: "string", description: "a particular version (optional)" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const module = String(args.module ?? "");
|
||||
if (!module) return { error: "name a module" };
|
||||
const found = await graph.show(module, args.commit ? String(args.commit) : undefined);
|
||||
return found ? { module: found } : { error: `the catalogue holds no ${module}` };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "catalog_provides",
|
||||
description:
|
||||
"What provides a provision, and what needs it. Answered from what modules declare about themselves, so it does not require holding every manifest at once.",
|
||||
input: { provision: { type: "string", description: "the provision, e.g. postgres-database" } },
|
||||
run: async (args) => {
|
||||
const provision = String(args.provision ?? "");
|
||||
if (!provision) return { error: "name a provision" };
|
||||
return await graph.whoProvides(provision);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "catalog_dependents",
|
||||
description:
|
||||
"What was built against this module — the modules a change to it reaches. Derived from builds, not from a declared list, so it cannot drift from what the code actually uses.",
|
||||
input: { module: { type: "string", description: "the module that would change" } },
|
||||
run: async (args) => {
|
||||
const module = String(args.module ?? "");
|
||||
if (!module) return { error: "name a module" };
|
||||
return { module, dependents: await graph.dependents(module) };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "catalog_stale",
|
||||
description:
|
||||
"What must be rebuilt and why — every module built against something that has since moved. `buildable` is the subset waiting on nothing that is itself stale, which is the set that can be built right now.",
|
||||
input: {},
|
||||
run: async () => ({
|
||||
stale: await graph.stale(),
|
||||
buildable: await graph.buildable(),
|
||||
}),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// Opened from the environment when the runtime asks for the module's tools. When it cannot be —
|
||||
// no DATABASE_URL — the module contributes no tools rather than taking the whole tool runtime down
|
||||
// with it, which is the postgres precedent.
|
||||
registerModuleTools("mesh-catalog", (env) => {
|
||||
try {
|
||||
return getCatalogueTools(Graph.fromEnv(env));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": [
|
||||
"pg.d.ts",
|
||||
"store.ts",
|
||||
"index.ts",
|
||||
"tools/index.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# postgres's runtime: the tool runtime, carrying this module's compiled provisioner, tools and
|
||||
# event consumer.
|
||||
#
|
||||
# **Built from this module's own directory and nothing else.** The sdk is in the base image, so
|
||||
# nothing is copied out of a neighbouring checkout — which is what lets the mesh build this from a
|
||||
# repository and a path (novox/hq ADR 0069) rather than only on a workstation that happens to have
|
||||
# the siblings.
|
||||
#
|
||||
# The base is named by ARG so it can be pinned to a digest the mesh's registry assigned. A tag would
|
||||
# make this runtime's contents depend on what somebody last pushed under that name.
|
||||
ARG RUNTIME_BASE=127.0.0.1:5000/mesh-tools/runtime@sha256:44b5d8bc30107fdc3bffdaebc1ca2de97615053853f826dfccce253a01a163fd
|
||||
|
||||
FROM ${RUNTIME_BASE} AS build
|
||||
# Compiled under /app/modules so `@novox/mesh-sdk` resolves upward into the base's own
|
||||
# node_modules — the module is compiled against exactly the sdk it will run against.
|
||||
WORKDIR /app/modules/postgres
|
||||
COPY . .
|
||||
# The compiler is invoked by its real path rather than through node_modules/.bin, whose entries are
|
||||
# symlinks to a launcher that requires its library relatively — resolved away when the base image
|
||||
# was assembled.
|
||||
RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts provisioner/index.ts tools/index.ts \
|
||||
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
|
||||
|
||||
FROM ${RUNTIME_BASE}
|
||||
# **This module talks to its database through psql, so psql has to be here.** The client is how
|
||||
# postgres's provisioner runs DDL — it does not carry a driver — and the runtime base holds only
|
||||
# what every module needs.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends postgresql-client \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY --from=build /app/modules/postgres/dist /app/modules/postgres/dist
|
||||
# What a tool host should load from this module: its event consumer and its tools, which are
|
||||
# separate entrypoints because they are loaded by different things. The provisioner is the third,
|
||||
# and is not listed here — the declaration names it in the container's `args`, because it is what
|
||||
# this module's own container runs. One image, because they are one module and share a client.
|
||||
ENV MESH_TOOL_MODULES=/app/modules/postgres/dist/index.js,/app/modules/postgres/dist/tools/index.js
|
||||
@@ -102,7 +102,6 @@
|
||||
"id": "runtime",
|
||||
"type": "container",
|
||||
"name": "mesh-postgres",
|
||||
"image": "mesh-runtime-postgres@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"network": "postgres",
|
||||
"volumes": [
|
||||
"/var/lib/mesh/postgres/broker:/run/secrets/broker:ro",
|
||||
@@ -114,7 +113,21 @@
|
||||
"MESH_PROVISION_PASSWORD_FILE": "/run/secrets/superuser",
|
||||
"MESH_BROKER_FILE": "/run/secrets/broker",
|
||||
"MESH_RECEIVES": "/var/lib/postgres/grants/mesh.json"
|
||||
}
|
||||
},
|
||||
"artifact": "runtime",
|
||||
"args": [
|
||||
"run",
|
||||
"/app/modules/postgres/dist/provisioner/index.js"
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user