Feature 57 (config and provisioning): modules declare config and provisions, their files rendered from templates; synced with main (issues of feature 57, 874)

This commit is contained in:
2026-10-16 07:37:43 +00:00
parent 1a2c917abf
commit 9652b2d5ce
771 changed files with 52063 additions and 1774 deletions
+205 -84
View File
@@ -40,10 +40,12 @@ import { emit, on } from "@novox/mesh-sdk/events";
import { registerModuleTools } from "@novox/mesh-sdk/tools";
import { commitStatusTool } from "./delivery.js";
import { mergeTool } from "./merge.js";
import { GiteaClient, movedSince } from "./client.js";
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
type RepoCheckFacts, reposToLook } from "./pulls.js";
repoCheckFacts, reposToLook } from "./pulls.js";
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
import { keepSiblings, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
@@ -135,6 +137,7 @@ async function pollMerged(client: GiteaClient): Promise<void> {
await emit("pull.closed", { owner: repo.owner, repo: repo.name, number: pull.number, title: pull.title,
head: pull.head, head_sha: pull.head_sha, base: pull.base, html_url: pull.html_url });
console.log(`[gitea] announced ${repo.full_name}#${pull.number} closed unmerged`);
await siblingMoves(client, repo.full_name, pull.number);
}
announced.add(closedKey);
changed = true;
@@ -145,57 +148,9 @@ async function pollMerged(client: GiteaClient): Promise<void> {
// at once.
const fresh = !!pull.merged_at && !!since && pull.merged_at > since;
if (primedMerges && fresh) {
// What it changed, asked for only now: a module is rebuilt because a file inside its own
// directory moved, and without this every module built from a repository is rebuilt for a
// change to any of them (novox/hq 04-ISSUES/131).
const changed = await client.listPullFiles(repo.owner, repo.name, pull.number);
// Which of the directories they are in hold a module at the merge commit (novox/hq issue 278): a
// change inside one is that module's, held or not, and only a file in none is shared code. Not
// said when the list is cut or the forge could not be asked; the mesh then keeps its old rule.
let moduleDirs: string[] | null = null;
if (!changed.truncated) {
try {
moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, pull.merge_commit_sha, changed.paths);
} catch (err) {
console.error(`[gitea] ${repo.full_name}#${pull.number}: which directories hold a module could not be read, ` +
`so the mesh reads its files by the old rule — ${err instanceof Error ? err.message : String(err)}`);
}
}
// The head it merged, and what its head was checked as (novox/hq ADR 0239): the delivery it was, made
// from the forge's word when its owner never heard the head.
let headChecks: Record<string, string> | null = null;
if (pull.head_sha) {
try {
headChecks = await client.commitStatuses(repo.owner, repo.name, pull.head_sha);
} catch (err) {
console.error(`[gitea] ${repo.full_name}#${pull.number}: its head's statuses could not be read — ${err instanceof Error ? err.message : err}`);
}
}
await emit("pull.merged", {
owner: repo.owner,
repo: repo.name,
number: pull.number,
title: pull.title,
body: pull.body,
head_sha: pull.head_sha,
...(headChecks ? { head_checks: headChecks } : {}),
head: pull.head,
base: pull.base,
merge_commit_sha: pull.merge_commit_sha,
merged_at: pull.merged_at,
clone_url: repo.clone_url,
html_url: pull.html_url,
paths: changed.paths,
paths_truncated: changed.truncated,
// Which of them the merge deleted (novox/hq ADR 0236): a module whose manifest went is forgotten,
// not built.
removed: changed.removed,
...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}),
});
// Said, because a trigger that fires silently is indistinguishable from one that did not
// fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told.
console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`);
if (await announceMerge(client, repo, pull)) pace.miss(`the merge of ${repo.full_name}#${pull.number}`);
}
else announced.add(pull.merge_commit_sha);
announced.add(pull.merge_commit_sha);
changed = true;
}
@@ -206,6 +161,75 @@ async function pollMerged(client: GiteaClient): Promise<void> {
lastLook = began;
}
type RepoOf = { owner: string; name: string; full_name: string; clone_url?: string };
type PullOf = Awaited<ReturnType<GiteaClient["listPullRequests"]>>[number];
/** **A merge, announced once, whichever path saw it first** (novox/hq issue 729): the poll or the forge's event. */
async function announceMerge(client: GiteaClient, repo: RepoOf, pull: PullOf): Promise<boolean> {
if (!pull.merged || !pull.merge_commit_sha || announced.has(pull.merge_commit_sha)) return false;
// Recorded before the first wait, so the other path finds it taken while this one is still asking the forge.
announced.add(pull.merge_commit_sha);
try {
// What it changed, asked for only now: a module is rebuilt because a file inside its own
// directory moved, and without this every module built from a repository is rebuilt for a
// change to any of them (novox/hq 04-ISSUES/131).
const changed = await client.listPullFiles(repo.owner, repo.name, pull.number);
// Which of the directories they are in hold a module at the merge commit (novox/hq issue 278): a
// change inside one is that module's, held or not, and only a file in none is shared code. Not
// said when the list is cut or the forge could not be asked; the mesh then keeps its old rule.
let moduleDirs: string[] | null = null;
if (!changed.truncated) {
try {
moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, pull.merge_commit_sha, changed.paths);
} catch (err) {
console.error(`[gitea] ${repo.full_name}#${pull.number}: which directories hold a module could not be read, ` +
`so the mesh reads its files by the old rule — ${err instanceof Error ? err.message : String(err)}`);
}
}
// The head it merged, and what its head was checked as (novox/hq ADR 0239): the delivery it was, made
// from the forge's word when its owner never heard the head.
let headChecks: Record<string, string> | null = null;
if (pull.head_sha) {
try {
headChecks = await client.commitStatuses(repo.owner, repo.name, pull.head_sha);
} catch (err) {
console.error(`[gitea] ${repo.full_name}#${pull.number}: its head's statuses could not be read — ${err instanceof Error ? err.message : err}`);
}
}
await emit("pull.merged", {
owner: repo.owner,
repo: repo.name,
number: pull.number,
title: pull.title,
body: pull.body,
head_sha: pull.head_sha,
...(headChecks ? { head_checks: headChecks } : {}),
head: pull.head,
base: pull.base,
merge_commit_sha: pull.merge_commit_sha,
merged_at: pull.merged_at,
clone_url: repo.clone_url,
html_url: pull.html_url,
paths: changed.paths,
paths_truncated: changed.truncated,
// Which of them the merge deleted (novox/hq ADR 0236): a module whose manifest went is forgotten,
// not built.
removed: changed.removed,
...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}),
});
// Said, because a trigger that fires silently is indistinguishable from one that did not
// fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told.
console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`);
// Its own merge of the head that was read is no sibling's move; any other is (ADR 0382 §5).
await siblingMoves(client, repo.full_name, pull.number);
} catch (err) {
announced.delete(pull.merge_commit_sha);
throw err;
}
keepAnnounced();
return true;
}
// **Every new head of an open pull request is announced, once** (novox/hq to-be 45 §9): the mesh checks it
// against every machine of its facts before it merges. Kept beside the merges' record, so a restart
// announces nothing twice; the first look on a machine with no record announces only what moved in the
@@ -228,6 +252,33 @@ function keepHeads(): void {
writeFileSync(tmp, JSON.stringify({ heads }));
renameSync(tmp, pullsRecord);
}
/** **A head, announced once, whichever path saw it first** (novox/hq issue 729): the poll or the forge's event. */
async function announceHead(client: GiteaClient, repo: RepoOf, pull: OpenOf): Promise<boolean> {
const key = `${repo.full_name}#${pull.number}`;
const sha = String(pull.head_sha ?? "");
const before = heads[key];
// Claimed before the first wait, so the other path finds it taken while this one is still announcing.
if (!claimHead(heads, key, sha)) return false;
try {
// The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5).
await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url ?? "", pull));
} catch (err) {
if (before === undefined) delete heads[key];
else heads[key] = before;
throw err;
}
keepHeads();
// A sibling's new head puts out of date every verdict that read the sibling at another head (ADR 0382 §5).
await siblingMoves(client, repo.full_name, pull.number);
// Said, so an operator knows the mesh was asked to check it.
console.log(`[gitea] announced ${key} at ${sha.slice(0, 8)} to be checked before it merges`);
// Pending until the verdict comes, so the pull request says a check is running rather than nothing — and
// where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446).
await gates.pending(repo.owner, repo.name, sha);
return true;
}
type OpenOf = Awaited<ReturnType<GiteaClient["listOpenPullsSince"]>>[number];
let lastPullLook = "";
// **A sweep now and then looks at every repository** (novox/hq issue 347), whatever moved: the floor is a
// heuristic, and a heuristic that misses is caught by the next sweep rather than never.
@@ -259,15 +310,8 @@ async function pollPulls(client: GiteaClient): Promise<void> {
const key = `${repo.full_name}#${pull.number}`;
const fresh = primedPulls || (!!pull.updated_at && pull.updated_at > dayAgo);
if (fresh) {
// The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5).
await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url, pull));
// Said, so an operator knows the mesh was asked to check it.
console.log(`[gitea] announced ${key} at ${String(pull.head_sha).slice(0, 8)} to be checked before it merges`);
// Pending until the verdict comes, so the pull request says a check is running rather than nothing — and
// where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446).
await gates.pending(repo.owner, repo.name, String(pull.head_sha));
}
heads[key] = String(pull.head_sha);
if (await announceHead(client, repo, pull)) pace.miss(`${key} at ${String(pull.head_sha).slice(0, 8)}`);
} else heads[key] = String(pull.head_sha);
changed = true;
}
if (await trunksMoved(client, repo.owner, repo.name)) changedTrunks = true;
@@ -303,7 +347,22 @@ function keepTrunks(): void {
/** Whether a trunk of one repository moved, weighed with this module's forge and its one writer (trunk.ts). */
function trunksMoved(client: GiteaClient, owner: string, name: string): Promise<boolean> {
return weighTrunks(client, gates, trunks, owner, name);
return weighTrunks(client, gates, trunks, owner, name, () => siblingMoves(client, `${owner}/${name}`));
}
// **The siblings each open pull request's latest verdict names** (novox/hq ADR 0382 §5, siblings.ts), by member, kept beside
// the other records. A sibling's new head, close unmerged, merge, or a move of its base puts the merge check of every
// member whose verdict names it back to pending, with the reason; nothing is run for it.
const siblingsRecord = siblingsFile(process.env.MESH_GITEA_STATE_DIR);
const siblingNamed = readSiblings(siblingsRecord);
/** A sibling moved, weighed (ADR 0382 §5): never fatal to what heard it. */
async function siblingMoves(client: GiteaClient, repository: string, number?: number): Promise<void> {
try {
if (await weighSiblings(client, gates, siblingNamed, repository, number)) keepSiblings(siblingsRecord, siblingNamed);
} catch (err) {
console.error(`[gitea] whether a sibling of ${repository}${number ? "#" + number : ""} moved could not be weighed — ${err instanceof Error ? err.message : err}`);
}
}
// **The verdict, set where the pull request shows it.** Every verdict is the head commit's status; one
@@ -325,6 +384,17 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
console.error("[gitea] a merge check's verdict named no repository, commit or verdict; ignored");
return;
}
// The siblings this verdict read are kept for the member (ADR 0382 §5); a verdict that read none removes the record.
const siblings = siblingsOf(c.siblings);
if (c.number && noteVerdict(siblingNamed, `${c.owner}/${c.repo}#${c.number}`.toLowerCase(), c.commit, siblings)) {
keepSiblings(siblingsRecord, siblingNamed);
}
// A sibling that moved while the check ran is weighed once this verdict has landed, in the head's own turn after it.
if (siblings.length) {
for (const repository of new Set(siblings.map((s) => s.repository))) {
void Promise.resolve().then(() => siblingMoves(client, repository));
}
}
// Each status links to the pull request, where the delivery's view is kept (novox/hq ADR 0239).
let target: string | undefined;
let base: string | undefined;
@@ -362,22 +432,6 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
}
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
// required would let an untested repository merge.
async function repoCheckFacts(client: GiteaClient, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
if (c["repo-check"]?.verdict !== "warning") return {};
const defined = await client.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
if (defined !== false) return { defined };
const rules = await client.branchProtections(c.owner, c.repo).catch(() => undefined);
if (!rules) return { defined };
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === (base || "main"));
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
return { defined, required };
}
// **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words,
// where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last
// (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending.
@@ -391,10 +445,16 @@ const gates = new GateWriter({
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
// reach, nothing is served, as the tools' bundle serves nothing then.
registerModuleTools("gitea", () => (gitea ? [commitStatusTool({
/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what
* gitea_events says of them (webhook.ts). */
const pace = new Pace();
if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
gates,
}), mergeTool(gitea, { gates })] : []));
}), mergeTool(gitea, { gates })] : [])]);
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
@@ -413,12 +473,70 @@ async function buildStarted(event: { body: unknown }): Promise<void> {
}
}
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
const b = event.body as { owner?: string; repo?: string; number?: number };
if (!b.owner || !b.repo || !b.number) return;
if (event.name === "forge.pull-request.closed") {
// Closed unmerged is a sibling's move; a merge is weighed with its announcement below (ADR 0382 §5).
if (!(event.body as { merged?: boolean }).merged) await siblingMoves(client, `${b.owner}/${b.repo}`, b.number);
return;
}
const head = ["forge.pull-request.opened", "forge.pull-request.synchronized", "forge.pull-request.reopened"].includes(event.name);
const merged = event.name === "forge.pull-request.merged";
if (!head && !merged) return;
const [repo, pull] = await Promise.all([client.getRepo(b.owner, b.repo), client.getPullRequest(b.owner, b.repo, b.number)]);
if (head && pull.state === "open") await announceHead(client, repo, pull);
if (merged) await announceMerge(client, repo, pull);
}
/**
* **The receiver, and the forge's one system webhook pointing at it** (issue 729). The secret is minted and kept beside
* the token; the webhook is made or corrected with the admin account (ADR 0266: the tools' token holds no admin scope).
* Answers whether events are heard; a forge that refuses is said, and the polls keep their pace.
*/
async function listenForForge(client: GiteaClient): Promise<boolean> {
const stateDir = process.env.MESH_GITEA_STATE_DIR;
const url = process.env.MESH_GITEA_HOOK_URL;
if (!stateDir || !url) {
console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false);
return false;
}
try {
const secret = keptSecret(join(stateDir, "webhook.secret"));
const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0";
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
refused: (reason, from) => pace.refused(reason, from),
publish: (event) => emit(event.name, event.body),
act: (event) => actOn(client, event) });
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(port, host, () => resolve());
});
const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
pace.listening(host, port, hook.url, hook);
console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` +
`at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` +
`${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` +
`${hook.events.length} event families, signed`);
return true;
} catch (err) {
const why = err instanceof Error ? err.message : String(err);
pace.unheard(why);
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`);
return false;
}
}
if (gitea) {
const client = gitea;
// A poll that fails says so once, not once a minute: the same reason repeating (the forge not up
// yet, the admin account refused on a restored forge) is one fact, and a recovery is worth a line.
let failing: string | null = null;
const tick = (fn: () => Promise<void>, everyMs: number): void => {
const tick = (fn: () => Promise<void>, every: number | (() => number)): void => {
const everyMs = (): number => (typeof every === "number" ? every : every());
// **One pass at a time** (novox/hq issue 250): the next pass is scheduled when this one has ended, so a
// pass that outlasts its interval delays the next instead of running beside it — two passes at once
// could each announce the same merge before either recorded it.
@@ -433,12 +551,15 @@ if (gitea) {
if (why !== failing) console.error(`[gitea] not watching until this clears — ${why}`);
failing = why;
})
.finally(() => setTimeout(run, everyMs));
.finally(() => setTimeout(run, everyMs()));
run();
};
// **The forge's events first, the polls as the safety net** (novox/hq issue 729): while verified deliveries arrive the
// polls slow to every five minutes and announce only what no event said (Pace); otherwise they keep 30 s.
await listenForForge(client);
tick(() => pollRepos(client), 60_000);
tick(() => pollMerged(client), 30_000);
tick(() => pollPulls(client), 30_000);
tick(() => pollMerged(client), () => pace.every());
tick(() => pollPulls(client), () => pace.every());
await on("mesh-controller.checked", (event) => {
const c = (event.body ?? {}) as Checked;
// A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts).