From de86f8123dc14a0f031587ce6443e05cb21a9a29 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:36:13 +0100 Subject: [PATCH] mesh-catalog: take builds in from the builder's log through builder_builds_since and its done keys, not the controller's built-before (issue 1955, ADR 0571 clause 9) --- modules/mesh-catalog/heard.ts | 73 ++++++++++++++++++-- modules/mesh-catalog/index.ts | 92 +++++++++++++++++++++---- modules/mesh-catalog/module.json | 13 ++-- modules/mesh-catalog/test/heard.test.ts | 27 +++++--- 4 files changed, 171 insertions(+), 34 deletions(-) diff --git a/modules/mesh-catalog/heard.ts b/modules/mesh-catalog/heard.ts index eaa8f882..619639ed 100644 --- a/modules/mesh-catalog/heard.ts +++ b/modules/mesh-catalog/heard.ts @@ -45,13 +45,72 @@ export interface Built { branches?: string[]; } -/** The events a build outcome reaches this module on: the build seat's, as it happens, and the control plane's replay. */ -export const BUILD_EVENTS = [ - // The build seat node-build-agent and the retired mesh-build-machine went with build-agent (novox/hq ADR 0524 §5, - // ADR 0525 §10): no build seat says `built` since R4g, so what the mesh holds is asked when this module starts. - // What the mesh already held, when this module asks what it missed (novox/hq ADR 0134). - "mesh-controller.built-before", -] as const; +/** + * Where a build outcome comes from now: the builder's log (novox/hq ADR 0571 clauses 1 and 9). The module asks the + * builder's tool for the builds that ended since a time, and again whenever the builder's state of done keys changes. + */ +export const BUILDER = "mesh-delivery-builder"; +export const BUILDS_SINCE_TOOL = "builder_builds_since"; +export const BUILDS_STATE = "mesh-delivery-builder.builds"; +export const MANIFESTS_STATE = "mesh-delivery-indexer.manifests"; + +/** One entry of the builder's log, an ended build (mesh-sdk delivery.BuildLogEntry). */ +export interface LogEntry { + event: string; + module: string; + fp: string; + repository: string; + directory: string; + commit: string; + ref?: string; + node?: string; + at: string; + /** done, failed or queued. */ + outcome?: string; + /** The digest of the manifest built, read from the indexer's state of manifests. */ + manifest?: string; + against?: string[]; + made?: Made[]; +} + +/** The answer of the builder's tool: every ended build since a time, oldest first, and whether the log held more. */ +export interface BuildsSince { + since: string; + builds: LogEntry[]; + more?: boolean; +} + +/** + * What a tool answers, whether bare or wrapped as the runtime wraps a tool's reply ({output} or {content:[{text}]}). + */ +export function answerOf(raw: unknown): T { + const r = raw as { output?: unknown; content?: { text?: string }[] } | null; + if (r && typeof r.output === "string") return JSON.parse(r.output) as T; + if (r && Array.isArray(r.content) && typeof r.content[0]?.text === "string") return JSON.parse(r.content[0].text) as T; + if (typeof raw === "string") return JSON.parse(raw) as T; + return raw as T; +} + +/** + * A log entry as the build outcome the catalogue places: a build that is done is placed with its manifest, what it + * made and what it was built against; one that failed is passed over (placeable says so); one still queued has not + * ended and is not taken in. `replay` is set by the caller for history. + */ +export function builtFrom(e: LogEntry, manifest: unknown, replay: boolean): Built | undefined { + if (e.outcome === "queued") return undefined; + return { + module: e.module, + commit: e.commit, + repository: e.repository, + path: e.directory, + ref: e.ref ?? "", + manifest, + made: e.made ?? [], + against: e.against ?? [], + replay, + ...(e.outcome === "failed" ? { failed: "the builder's log says the build failed" } : {}), + }; +} /** * Whether a build outcome is a module-version to place in the graph, and why not when it is not. diff --git a/modules/mesh-catalog/index.ts b/modules/mesh-catalog/index.ts index 13a98d2d..cc034002 100644 --- a/modules/mesh-catalog/index.ts +++ b/modules/mesh-catalog/index.ts @@ -9,9 +9,15 @@ // and appear the next time round, once whatever they were waiting for is registered. A chain and a // diamond need no special handling, and nothing holds a plan. -import { on, emit } from "@novox/mesh-sdk/events"; +import { emit } from "@novox/mesh-sdk/events"; +import { broker } from "@novox/mesh-sdk/messaging"; +import { invokeTool } from "@novox/mesh-sdk/tools"; +import { state } from "@novox/mesh-sdk/state"; import { Graph } from "./store.js"; -import { BUILD_EVENTS, placeable, type Built } from "./heard.js"; +import { + answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE, + type Built, type BuildsSince, type LogEntry, +} from "./heard.js"; const graph = Graph.fromEnv(); @@ -77,18 +83,74 @@ const placeTheBuild = async (event: { body: unknown }): Promise => { } }; -// As it happens — from the build seat that builds now, and the retired one while it still answers — and -// what the mesh already held when this module asked what it missed (novox/hq issue 712). -for (const heard of BUILD_EVENTS) await on(heard, placeTheBuild); +// **From the builder's log** (novox/hq ADR 0571 clauses 1 and 9). The builder keeps every build's end in a log of its +// own and answers the builds that ended since a time by its tool. This module asks at its start for what it missed (all +// the log holds, taken in as history: registered, announced to nobody), and again whenever the builder's state of done +// keys changes, for what ended since the last build it took in. Registering a build already held changes nothing, and a +// build is taken in once per run of this module. +const taken = new Set(); +let since = new Date(0).toISOString(); +let caughtUp = false; +let asking: Promise | undefined; +let askAgain = false; -// **And ask for what was built before this catalogue existed** (novox/hq 04-ISSUES/050). -// -// The queue above is durable, so nothing is missed once this is running. What it cannot have is -// what was announced before it first ran — and on a fresh mesh that is never arbitrary: the shared -// base, the store this runs on, and this module itself are each necessarily built BEFORE a -// catalogue exists to hear about them. The graph's foundation is the part it never sees. -// -// Asked on every start, not only the first. A catalogue cannot tell whether it has a gap, and the -// answer is idempotent: registering a build already held changes nothing and announces nothing. -// Asked AFTER subscribing, so a build arriving during the replay is not lost between the two. +const manifestOf = async (digest: string | undefined): Promise => { + if (!digest) return {}; + const held = await state<{ manifest?: string }>(MANIFESTS_STATE).get(digest); + if (!held?.value.manifest) return {}; + return JSON.parse(held.value.manifest); +}; + +const takeIn = async (e: LogEntry, replay: boolean): Promise => { + const key = `${e.module}/${e.fp}`; + if (taken.has(key)) return; + const built = builtFrom(e, await manifestOf(e.manifest), replay); + if (!built) return; + await placeTheBuild({ body: built }); + taken.add(key); +}; + +const catchUp = async (): Promise => { + for (;;) { + const answer = answerOf( + await invokeTool(broker(), BUILDER, BUILDS_SINCE_TOOL, { since }), + ); + for (const e of answer.builds ?? []) { + await takeIn(e, !caughtUp); + if (e.at > since) since = e.at; + } + if (!answer.more) break; + } + caughtUp = true; +}; + +/** One ask at a time; a change heard while asking is asked for again once, so none is missed. */ +const ask = (): Promise => { + if (asking) { + askAgain = true; + return asking; + } + asking = (async () => { + try { + do { + askAgain = false; + await catchUp(); + } while (askAgain); + } catch (err) { + console.error("mesh-catalog: the builder's builds since", since, "could not be taken in:", err); + } finally { + asking = undefined; + } + })(); + return asking; +}; + +// Asked AFTER watching begins, so a build ending while the first ask runs asks again rather than being missed: the watch +// hands every current key first (the start's ask), then every change. +await state(BUILDS_STATE).watch(async (change) => { + if (!change.current) await ask(); +}); +await ask(); + +// **And say that this catalogue has caught up** (novox/hq 04-ISSUES/050). await emit("catching-up", {}); diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json index b8f45096..c5ad61a4 100644 --- a/modules/mesh-catalog/module.json +++ b/modules/mesh-catalog/module.json @@ -25,9 +25,7 @@ "secrets": { "postgres-database": "${dir:state}/database.secret" }, - "consumes": [ - "mesh-controller.built-before" - ], + "consumes": [], "emits": [ "registered", "upgraded", @@ -92,5 +90,12 @@ "from": "files" } ] - } + }, + "invokes": [ + "mesh-delivery-builder.builder_builds_since" + ], + "reads": [ + "mesh-delivery-builder.builds", + "mesh-delivery-indexer.manifests" + ] } diff --git a/modules/mesh-catalog/test/heard.test.ts b/modules/mesh-catalog/test/heard.test.ts index 84b91ac4..391cb5e9 100644 --- a/modules/mesh-catalog/test/heard.test.ts +++ b/modules/mesh-catalog/test/heard.test.ts @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; -import { BUILD_EVENTS, placeable } from "../heard.js"; +import { answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE } from "../heard.js"; // A walk's build as the build seat says it (mesh-controller's link.BuildResult), here mailu's of novox/hq issue 712. const walkBuild = { @@ -20,14 +20,25 @@ const walkBuild = { made: [{ name: "code", kind: "bundle", reference: "artifact-store://mailu/code/blobs/sha256:d041" }], }; -test("the catalogue hears no retired build seat, and what the mesh held (issue 712, issue 1887 E2)", async () => { - for (const retired of ["node-build-agent.built", "mesh-build-machine.built"]) { - assert.ok(!(BUILD_EVENTS as readonly string[]).includes(retired), `${retired} is still heard`); - } - assert.ok(BUILD_EVENTS.includes("mesh-controller.built-before")); - // Subscribing is not enough: the mesh grants and routes only what the manifest consumes. +test("the catalogue takes builds from the builder's log, and hears no controller event (ADR 0571 clause 9)", async () => { const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); - for (const heard of BUILD_EVENTS) assert.ok(manifest.consumes.includes(heard), `${heard} is not consumed`); + assert.deepEqual(manifest.consumes, []); + assert.ok(manifest.invokes.includes(`${BUILDER}.${BUILDS_SINCE_TOOL}`), "the builder's tool is not invoked"); + for (const read of [BUILDS_STATE, MANIFESTS_STATE]) assert.ok(manifest.reads.includes(read), `${read} is not read`); + // A log entry is placed as a build: done with its manifest, made and against; failed passed over; queued not taken in. + const entry = { event: "build.ended", module: "mailu", fp: "fp1", repository: walkBuild.repository, directory: "modules/mailu", + commit: walkBuild.commit, ref: "main", at: "2026-10-29T20:00:00Z", outcome: "done", manifest: "ab", against: ["img@sha256:1"], + made: walkBuild.made }; + const done = builtFrom(entry, walkBuild.manifest, false); + assert.deepEqual(placeable(done!), { placed: "yes", module: "mailu", commit: walkBuild.commit }); + assert.deepEqual([done!.path, done!.against, done!.replay], ["modules/mailu", ["img@sha256:1"], false]); + assert.equal(placeable(builtFrom({ ...entry, outcome: "failed" }, {}, true)!).placed, "no"); + assert.equal(builtFrom({ ...entry, outcome: "queued" }, {}, true), undefined); + // The tool's answer is read bare or as the runtime wraps it. + const bare = { since: "x", builds: [entry] }; + for (const raw of [bare, { output: JSON.stringify(bare) }, { content: [{ text: JSON.stringify(bare) }] }]) { + assert.equal(answerOf(raw).builds[0].module, "mailu"); + } }); test("a walk's build is placed, under its module and commit", () => {