From f0fbe193fa3c36afe633588996f25838b13ad513 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 17:59:34 +0100 Subject: [PATCH 01/28] gitea: mesh-tools administers a user's delivered repository and puts main's rule right, permissions read as the account sees them; knowledge-base.progress: minimal answers leave items to the tree, failed calls warn before parse faults (issues 1945, 1947) --- modules/gitea/client.ts | 48 ++++++++++-- modules/gitea/test/tools-account.test.ts | 31 +++++++- modules/gitea/writes.ts | 77 +++++++++++++------ .../cmd/mesh-knowledge/progress.go | 15 ++++ .../cmd/mesh-knowledge/progress_test.go | 11 +++ 5 files changed, 151 insertions(+), 31 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index a36e113f..f63a4456 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -6,7 +6,7 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js"; -import { MAIN_BRANCH, mainRuleBody, protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; +import { MAIN_BRANCH, mainMergers, mainRuleBody, protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; import { Refused, checkSentPath, ruleCovers, pathSegment, pathSegments } from "./guard.js"; import { isCommit } from "./checks.js"; @@ -1047,12 +1047,19 @@ export class GiteaAdmin { return true; } - /** What the forge itself says an account may do on a repository: none, read, write, admin or owner. */ + /** What the forge gives an account on a repository, effective: none, read, write, admin or owner. It is the repository as + * that account sees it (the site administrator acts as the account with `sudo`), so access through an organisation's team + * counts. The collaborator endpoint is not used: it names only a collaborator's own permission, and says "read" of an + * account that writes through a team (novox/hq issue 1945). */ async repositoryPermission(owner: string, repo: string, user: string): Promise { - const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}/permission`; + const path = `${repoPath(owner, repo)}?sudo=${encodeURIComponent(user)}`; const res = await this.request(path); + if (res.status === 404) return "none"; if (res.status !== 200) GiteaAdmin.fail(path, res); - return String(res.body?.permission ?? "none"); + const p = res.body?.permissions; + if (p?.admin === true) return res.body?.owner?.login === user ? "owner" : "admin"; + if (p?.push === true) return "write"; + return p?.pull === true ? "read" : "none"; } /** Make an account a collaborator of a repository that reads only. The forge asks a repository's administrator for @@ -1067,7 +1074,38 @@ export class GiteaAdmin { await this.addCollaborator(owner, repo, user, "write"); } - private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write"): Promise { + /** Make an account an administrator collaborator of a repository (writes.ts: a repository of a user that builds a + * registered module, where the tools' account must also put main's rule right). */ + async addAdminCollaborator(owner: string, repo: string, user: string): Promise { + await this.addCollaborator(owner, repo, user, "admin"); + } + + /** Put main's rule right in one repository: no push, the merge whitelist (the tools' account first), no status required, + * and no administrator override. A rule the operator already set that is stronger is kept, never made weaker (ADR 0266): + * a whitelist that holds the tools' account and only logins the wanted one names, more approvals, a push already refused. + * Answers the rule as the forge holds it after, read back. With the administrator's login. */ + async ensureMainRule(owner: string, repo: string, mergers: string[] = mainMergers()): Promise<{ created: boolean; rule: BranchProtection }> { + const rules = await this.branchProtections(owner, repo); + const own = rules.find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH); + const kept = own?.enable_merge_whitelist === true ? (own.merge_whitelist_usernames ?? []) : []; + const whitelist = kept.length > 0 && kept.includes(mergers[0]) && kept.every((u) => mergers.includes(u)) ? kept : mergers; + const body = { ...mainRuleBody(mergers), merge_whitelist_usernames: whitelist, block_admin_merge_override: true }; + const path = `${repoPath(owner, repo)}/branch_protections`; + if (own) { + const res = await this.request(`${path}/${MAIN_BRANCH}`, { method: "PATCH", body: JSON.stringify(body) }); + if (res.status !== 200) GiteaAdmin.fail(`${path}/${MAIN_BRANCH}`, res); + } else { + const covering = rules.filter((r) => ruleCovers(String(r.rule_name ?? r.branch_name ?? ""), MAIN_BRANCH)); + const approvals = Math.max(0, ...covering.map((r) => Number(r.required_approvals ?? 0))); + const res = await this.request(path, { method: "POST", body: JSON.stringify({ rule_name: MAIN_BRANCH, ...(approvals > 0 ? { required_approvals: approvals } : {}), ...body }) }); + if (res.status !== 201) GiteaAdmin.fail(path, res); + } + const after = (await this.branchProtections(owner, repo)).find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH); + if (!after) throw new Error(`main's rule of ${owner}/${repo} was written, but the forge does not list it`); + return { created: !own, rule: after }; + } + + private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write" | "admin"): Promise { const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}`; const res = await this.request(path, { method: "PUT", body: JSON.stringify({ permission }) }); if (res.status === 204 || res.status === 200) return; diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 80519429..23a30d67 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -28,6 +28,7 @@ interface Team { id: number; name: string; units: any; members: Set } interface ForgeRepo { owner: string; name: string; full: string; private: boolean; archived?: boolean; ownerVisibility: string; collaborators: Map } const forgeRepos: ForgeRepo[] = []; +const userRules = new Map(); const repoFaults = { refuse: new Set(), permission: new Set() }; function addRepo(full: string, isPrivate: boolean, ownerVisibility = "public"): ForgeRepo { const [owner, name] = full.split("/"); @@ -84,7 +85,7 @@ function fakeForge() { const server = createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://fake"); const p = url.pathname.replace(/^\/api\/v1/, ""); - const me = who(req); + const me = url.searchParams.has("sudo") && isAdmin(who(req)) ? url.searchParams.get("sudo") : who(req); calls.push(`${req.method} ${p} as ${me}`); if (me === null) return json(res, 401, { message: "unauthorized" }); @@ -168,6 +169,20 @@ function fakeForge() { if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false }); return json(res, 404, { message: "branch does not exist" }); } + // A repository of a user: its rules are its administrator's (a site administrator, or a collaborator with admin). + m = p.match(/^\/repos\/(?!novox\/)([^/]+)\/([^/]+)\/branch_protections(?:\/(.*))?$/); + if (m) { + const key = `${m[1]}/${m[2]}`; + const repo = forgeRepos.find((r) => r.full === key); + if (!repo || !["admin", "owner"].includes(permissionOf(me, repo))) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + if (req.method === "GET" && m[3] === undefined) return json(res, 200, userRules.has(key) ? [userRules.get(key)] : []); + if (req.method === "PATCH" && m[3] === "main") { + if (!userRules.has(key)) return json(res, 404, { message: "branch protection does not exist" }); + Object.assign(userRules.get(key), await read(req)); + return json(res, 200, userRules.get(key)); + } + if (req.method === "POST") { const b = await read(req); userRules.set(key, b); return json(res, 201, b); } + } // gitea's reqAdmin on the whole group: reading the rules is administration too. m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/(.*))?$/); if (m) { @@ -205,6 +220,7 @@ function fakeForge() { m = p.match(/^\/repos\/([^/]+)\/([^/]+)(?:\/collaborators\/([^/]+)(\/permission)?)?$/); if (m) { const repo = forgeRepos.find((r) => r.owner === decodeURIComponent(m![1]) && r.name === decodeURIComponent(m![2])); + if (repo && req.method === "GET" && !m[3] && url.searchParams.has("sudo") && repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" }); if (!repo || permissionOf(me, repo) === "none") return json(res, 404, { message: "repository does not exist" }); if (req.method === "GET" && !m[3]) { const mine = permissionOf(me, repo); @@ -214,7 +230,8 @@ function fakeForge() { if (req.method === "GET" && m[4]) { if (repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" }); if (!isAdmin(me) && me !== user) return json(res, 403, { message: "only the user or an administrator" }); - return json(res, 200, { permission: permissionOf(user, repo) }); + // As gitea answers: the collaborator's own permission, not what a team gives the account. + return json(res, 200, { permission: repo.collaborators.get(user) ?? "read" }); } if (req.method === "PUT" && !m[4]) { // gitea's reqAdmin: the repository's administration, which on a repository of another user the site administrator alone holds. @@ -608,7 +625,9 @@ test("the merging account is given write on each repository the indexer's module await writes.take({ key: "car-hunter-2", op: "put", value: { v: 1, repository: "jschoubben/car-hunter" } }); assert.equal(forge.calls.length, from, "a repository already written asks nothing more"); const repo = (full: string) => forgeRepos.find((x) => x.full === full)!; - assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "write"); + assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "admin", "a user's repository is administered, not only written"); + assert.deepEqual(userRules.get("jschoubben/car-hunter"), { rule_name: "main", enable_push: false, enable_merge_whitelist: true, + merge_whitelist_usernames: [TOOLS_ACCOUNT], enable_status_check: false, status_check_contexts: [], block_admin_merge_override: true }); assert.equal(repo("novox/hq").collaborators.size, 0, "the organisation's repository is written through its team, no collaborator"); assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, true); assert.deepEqual(said, [{ type: WRITABLE_EVENT, body: { owner: "jschoubben", repo: "car-hunter", account: TOOLS_ACCOUNT } }]); @@ -636,8 +655,14 @@ test("the merging account is given write on each repository the indexer's module assert.ok(!forge.calls.slice(puts).some((c) => c.startsWith("PUT ")), "no write is asked on a repository no module builds from"); assert.equal(repo("jschoubben/dotfiles").collaborators.get(TOOLS_ACCOUNT), "read"); repoFaults.refuse.clear(); + // A stronger rule the operator set (a whitelist of the tools' account alone, two approvals) is kept, never weakened. + userRules.set("jschoubben/snake", { rule_name: "main", enable_push: true, enable_merge_whitelist: true, + merge_whitelist_usernames: [TOOLS_ACCOUNT], required_approvals: 2, block_admin_merge_override: false }); const after = await grantTool.run({ repositories: ["jschoubben/snake"] }) as any; assert.deepEqual(after.granted, ["jschoubben/snake"]); + assert.deepEqual(after.main_rule.map((x: any) => [x.repository, x.created, x.rule.enable_push, x.rule.required_approvals, x.rule.block_admin_merge_override]), + [["jschoubben/snake", false, false, 2, true]]); + assert.equal(repo("jschoubben/snake").collaborators.get(TOOLS_ACCOUNT), "admin"); assert.deepEqual(((await unwritableTool.run({})) as any).unwritable, []); // A module gone from the indexer leaves its repository out of the delivered list. await writes.take({ key: "snake", op: "delete" }); diff --git a/modules/gitea/writes.ts b/modules/gitea/writes.ts index ae9d41a6..70ef948c 100644 --- a/modules/gitea/writes.ts +++ b/modules/gitea/writes.ts @@ -18,10 +18,14 @@ // `repo.writable` {owner, repo, account}: mesh-delivery-gate, which does not ask the operator's Approve of a pull request // whose merge the forge would refuse, reads that repository again on it. // +// **A repository of a user is administered, not only written** (the operator, 2026-10-26: "mesh-tools should get all the +// permissions I give him"): the account is an admin collaborator there and puts main's rule right (client.ts ensureMainRule). +// // Not taken away: a repository whose last module goes from the indexer keeps its collaborator until a hand removes it; // gitea_unwritable_repos and gitea_grant_write say the delivered list as it is now. import type { ForgeRepository } from "./client.js"; +import type { BranchProtection } from "./protection.js"; /** The indexer's state this module reads: one key per registered module, its value naming the repository. */ export const INDEXER_MODULES = "mesh-delivery-indexer.modules"; @@ -33,7 +37,8 @@ export interface WriteForge { everyRepository(): Promise; repository(owner: string, repo: string): Promise; repositoryPermission(owner: string, repo: string, user: string): Promise; - addWriteCollaborator(owner: string, repo: string, user: string): Promise; + addAdminCollaborator(owner: string, repo: string, user: string): Promise; + ensureMainRule(owner: string, repo: string): Promise<{ created: boolean; rule: BranchProtection }>; } /** One change of the indexer's modules state, as the SDK's state watch hands it. */ @@ -51,8 +56,12 @@ export interface Written { delivered: string[]; /** Made writable now, each read back from the forge. */ granted: string[]; - /** Written already (the organisation's team, or a collaborator that writes). */ + /** Administered already (a repository of a user), or written through the organisation's team. */ already: string[]; + /** Each repository of a user whose main's rule was put right (no push, the merge whitelist, no status, no administrator + * override), read back: what the forge holds for it now. */ + main_rule: { repository: string; created: boolean; rule: { enable_push?: boolean; merge_whitelist_usernames?: string[] | null; + block_admin_merge_override?: boolean; enable_status_check?: boolean; required_approvals?: number } }[]; /** The forge refused, in its own words; or the repository is the organisation's, whose team should write it. */ refused: { repository: string; why: string }[]; /** Repositories the account cannot write that are left so, and why: they build no registered module. */ @@ -60,6 +69,7 @@ export interface Written { } const WRITES = new Set(["write", "admin", "owner"]); +const ADMINS = new Set(["admin", "owner"]); const said = (err: unknown): string => (err instanceof Error ? err.message : String(err)); function split(full: string): [string, string] { @@ -118,7 +128,7 @@ export class DeliveredWrites { "(the watch is not granted or not answered); nothing is granted"); } const delivered = this.delivered(); - const result: Written = { account: this.account, delivered, granted: [], already: [], refused: [], left_read_only: [] }; + const result: Written = { account: this.account, delivered, granted: [], already: [], main_rule: [], refused: [], left_read_only: [] }; const named = only && only.length ? only.map((x) => String(x).trim()) : null; const targets: string[] = []; for (const full of named ?? delivered) { @@ -140,36 +150,55 @@ export class DeliveredWrites { result.refused.push({ repository: full, why: `its permission could not be read — ${said(err)}` }); continue; } - if (WRITES.has(permission)) { - this.writable.add(full); - result.already.push(full); - continue; - } if (owner === this.org) { + if (WRITES.has(permission)) { + this.writable.add(full); + result.already.push(full); + continue; + } // The organisation's repositories are written through its team (tools-account.ts), never a collaborator. result.refused.push({ repository: full, why: `a repository of ${this.org}, which ${this.account} writes through the ` + `team ${this.org}/${this.account}; the forge gives it "${permission}", so the team is not as it should be (it is ` + "set again at the next mint of the tools' token)" }); continue; } - try { - await this.forge.addWriteCollaborator(owner, name, this.account); - } catch (err) { - result.refused.push({ repository: full, why: said(err) }); - continue; + // A repository of a user: the account administers it (the operator, 2026-10-26: "mesh-tools should get all the + // permissions I give him"), so it also puts main's rule right, which only an administrator may. + let changed = false; + if (!ADMINS.has(permission)) { + try { + await this.forge.addAdminCollaborator(owner, name, this.account); + } catch (err) { + result.refused.push({ repository: full, why: said(err) }); + continue; + } + let after: string; + try { + after = await this.forge.repositoryPermission(owner, name, this.account); + } catch (err) { + result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` }); + continue; + } + if (!ADMINS.has(after)) { + result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` }); + continue; + } + changed = true; } - let after: string; try { - after = await this.forge.repositoryPermission(owner, name, this.account); + const { created, rule } = await this.forge.ensureMainRule(owner, name); + result.main_rule.push({ repository: full, created, rule: { enable_push: rule.enable_push, + merge_whitelist_usernames: rule.merge_whitelist_usernames, block_admin_merge_override: rule.block_admin_merge_override, + enable_status_check: rule.enable_status_check, required_approvals: rule.required_approvals } }); } catch (err) { - result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` }); - continue; - } - if (!WRITES.has(after)) { - result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` }); + result.refused.push({ repository: full, why: `${this.account} administers it, but main's rule could not be put right — ${said(err)}` }); continue; } this.writable.add(full); + if (!changed) { + result.already.push(full); + continue; + } result.granted.push(full); await this.emit(WRITABLE_EVENT, { owner, repo: name, account: this.account }).catch((err) => this.log(`${this.account} writes ${full} now, and that could not be said on the bus — ${said(err)}`)); @@ -230,12 +259,14 @@ export function writeTools(writes: () => DeliveredWrites): Tool[] { }, { name: "gitea_grant_write", - description: "Give the mesh's merging forge account (mesh-tools) write, as a collaborator, on every repository that " + - "builds a registered module and that it cannot write now, or on the repositories named (each must build one). " + + description: "Give the mesh's merging forge account (mesh-tools) administration, as a collaborator, on every repository " + + "of a user that builds a registered module, and put main's rule right there (no push, the merge whitelist, no status " + + "required, no administrator override; a stronger rule the operator set is kept), or on the repositories named (each " + + "must build one). The answer's main_rule says each rule as the forge holds it after, read back. " + "The module does this itself at its start and when a module of a new repository is registered; this is the same " + "act by hand, such as when a merge or a commit status was refused 403. A repository of the organisation novox " + "is written through its team and needs nothing. Answers what was granted (each read back from the forge), what " + - "was written already, what the forge refused in its own words, and which other repositories it left read-only " + + "was administered already, what the forge refused in its own words, and which other repositories it left read-only " + "and why (they build no registered module). Needs the site administrator's login, used for this grant alone.", input: { repositories: { type: "array", items: { type: "string" }, description: "owner/name of each repository; empty or absent, every delivered one" }, diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go index 5edd5aaf..99afc4ca 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go @@ -167,6 +167,8 @@ func (k *KB) Progress(a ProgressAsk) (*Progress, error) { if err != nil { return nil, err } + // A cut takes the warnings from the last: the calls that failed are said before the answers that did not parse. + callsFirst(p.Warnings) fitProgress(p, boundOf(a.Bound)) return p, nil } @@ -822,8 +824,21 @@ func movedSay(t *time.Time) string { return " · moved " + when(t) } +// callsFirst puts the warnings of calls that failed before those of answers that did not parse, in place. +func callsFirst(w []string) { + sort.SliceStable(w, func(i, j int) bool { + return !strings.Contains(w[i], " answered something else") && strings.Contains(w[j], " answered something else") + }) +} + // fitProgress keeps an answer within its bound: the items go first, then the tree's last lines, saying what was cut. +// A minimal answer never carries the items: its tree says the same, one line per milestone, so leaving them out is how +// the minimal form is made (the items of design 47 alone passed 8192 bytes) and no cut. A detailed answer's items, and +// everything else that goes, are said in cut. func fitProgress(p *Progress, bound int) { + if p.Detail == DetailMinimal && p.Tree != "" { + p.Items = nil + } if size(p) <= bound { return } diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go index b2895846..ad8347af 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go @@ -306,12 +306,23 @@ func TestProgressOfADesignMinimalAndDetailed(t *testing.T) { if len(raw) > DefaultBound || p.Detail != DetailMinimal || !strings.Contains(p.Source, "issue 1781") { t.Fatalf("%d bytes, %s, %s", len(raw), p.Detail, p.Source) } + if p.Cut != "" || len(p.Items) != 0 { + t.Errorf("a minimal answer leaves the items to its tree and is not a cut: cut %q, %d items", p.Cut, len(p.Items)) + } + warns := []string{"mesh-delivery.show a answered something else: x", "issue-tracker.get 1: refused", "the tracker did not name 2", "issue-tracker.get 3 answered something else: y"} + callsFirst(warns) + if strings.Join(warns, "|") != "issue-tracker.get 1: refused|the tracker did not name 2|mesh-delivery.show a answered something else: x|issue-tracker.get 3 answered something else: y" { + t.Errorf("call failures do not come first: %q", warns) + } if seats.asked["seat:issue-tracker.list"] != 1 || seats.asked["seat:mesh-delivery.deliveries"] != 0 || seats.asked["seat:issue-tracker.get"] != 0 { t.Fatalf("minimal asked %v", seats.asked) } if w := strings.Join(p.Warnings, "\n"); !strings.Contains(w, "the tracker neither answered nor named missing") || !strings.Contains(w, "973, 525") { t.Errorf("a row the tracker did not answer is not said: %v", p.Warnings) } + if big, _ := k.Progress(ProgressAsk{Of: "design 47", Bound: 8192}); big.Cut != "" || big.Tree != p.Tree { + t.Errorf("a minimal answer at bound 8192 is cut: %q", big.Cut) + } lines := strings.Split(p.Tree, "\n") if !strings.HasPrefix(lines[0], "✗ design 47 · Delivery: from a commit to delivered — ") { t.Fatalf("first line %q", lines[0]) From c85472285d7156ee0f90d8613a0409ad507fd7ba Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:23:42 +0100 Subject: [PATCH 02/28] gitea: main's rule read back must say what was written, since the forge silently drops a whitelist login that cannot write (issue 1945, review round 1) --- modules/gitea/client.ts | 7 +++++++ modules/gitea/test/tools-account.test.ts | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index f63a4456..5e9f102e 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -1102,6 +1102,13 @@ export class GiteaAdmin { } const after = (await this.branchProtections(owner, repo)).find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH); if (!after) throw new Error(`main's rule of ${owner}/${repo} was written, but the forge does not list it`); + // The forge answers a write it did not keep: it drops a whitelist login that cannot write the repository, silently. + const listed = after.merge_whitelist_usernames ?? []; + const wrong: string[] = []; + if (after.enable_push !== false) wrong.push(`push is ${after.enable_push === true ? "allowed" : "not refused"}`); + if (after.enable_merge_whitelist !== true || !listed.includes(mergers[0])) wrong.push(`the merge whitelist [${listed.join(", ")}] leaves out ${mergers[0]}`); + if (after.enable_status_check !== false) wrong.push("a status is still required"); + if (wrong.length > 0) throw new Error(`main's rule of ${owner}/${repo} was written, but the forge holds otherwise: ${wrong.join("; ")}`); return { created: !own, rule: after }; } diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 23a30d67..2f1cd6aa 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -176,12 +176,15 @@ function fakeForge() { const repo = forgeRepos.find((r) => r.full === key); if (!repo || !["admin", "owner"].includes(permissionOf(me, repo))) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); if (req.method === "GET" && m[3] === undefined) return json(res, 200, userRules.has(key) ? [userRules.get(key)] : []); + // As gitea does: a whitelist login that cannot write the repository is dropped, and the write still answered. + const keep = (b: any) => (Array.isArray(b.merge_whitelist_usernames) + ? { ...b, merge_whitelist_usernames: b.merge_whitelist_usernames.filter((u: string) => ["write", "admin", "owner"].includes(permissionOf(u, repo))) } : b); if (req.method === "PATCH" && m[3] === "main") { if (!userRules.has(key)) return json(res, 404, { message: "branch protection does not exist" }); - Object.assign(userRules.get(key), await read(req)); + Object.assign(userRules.get(key), keep(await read(req))); return json(res, 200, userRules.get(key)); } - if (req.method === "POST") { const b = await read(req); userRules.set(key, b); return json(res, 201, b); } + if (req.method === "POST") { const b = keep(await read(req)); userRules.set(key, b); return json(res, 201, b); } } // gitea's reqAdmin on the whole group: reading the rules is administration too. m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/(.*))?$/); @@ -664,6 +667,15 @@ test("the merging account is given write on each repository the indexer's module [["jschoubben/snake", false, false, 2, true]]); assert.equal(repo("jschoubben/snake").collaborators.get(TOOLS_ACCOUNT), "admin"); assert.deepEqual(((await unwritableTool.run({})) as any).unwritable, []); + // The forge drops a whitelist login that cannot write and still answers 200: main's rule is read back and that fails loud. + const admin = GiteaAdmin.fromEnv(env); + await assert.rejects(admin.ensureMainRule("jschoubben", "dotfiles"), + /main's rule of jschoubben\/dotfiles was written, but the forge holds otherwise: the merge whitelist \[\] leaves out mesh-tools/); + userRules.get("jschoubben/snake").enable_status_check = true; + userRules.get("jschoubben/snake").enable_push = true; + const held = await admin.ensureMainRule("jschoubben", "snake"); + assert.deepEqual([held.rule.enable_push, held.rule.enable_status_check, held.rule.merge_whitelist_usernames], [false, false, [TOOLS_ACCOUNT]]); + userRules.delete("jschoubben/dotfiles"); // A module gone from the indexer leaves its repository out of the delivered list. await writes.take({ key: "snake", op: "delete" }); assert.deepEqual(writes.delivered(), ["jschoubben/car-hunter", "novox/hq"]); From e1e5a8ff0dfc97d19a59250bb7efacf5ea116c44 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:34:03 +0100 Subject: [PATCH 03/28] gitea: mesh-tools writes a user's delivered repository and main's rule stays the operator's: only its merge whitelist gets mesh-tools, read back; what else blocks the mesh's merge is said (issue 1945, the operator's answer 2026-10-26 18:30) --- modules/gitea/client.ts | 68 +++++++-------- modules/gitea/test/tools-account.test.ts | 40 +++++---- modules/gitea/writes.ts | 101 ++++++++++++++++------- 3 files changed, 124 insertions(+), 85 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 5e9f102e..408c93af 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -6,7 +6,7 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js"; -import { MAIN_BRANCH, mainMergers, mainRuleBody, protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; +import { MAIN_BRANCH, mainRuleBody, protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; import { Refused, checkSentPath, ruleCovers, pathSegment, pathSegments } from "./guard.js"; import { isCommit } from "./checks.js"; @@ -1074,45 +1074,39 @@ export class GiteaAdmin { await this.addCollaborator(owner, repo, user, "write"); } - /** Make an account an administrator collaborator of a repository (writes.ts: a repository of a user that builds a - * registered module, where the tools' account must also put main's rule right). */ - async addAdminCollaborator(owner: string, repo: string, user: string): Promise { - await this.addCollaborator(owner, repo, user, "admin"); - } - - /** Put main's rule right in one repository: no push, the merge whitelist (the tools' account first), no status required, - * and no administrator override. A rule the operator already set that is stronger is kept, never made weaker (ADR 0266): - * a whitelist that holds the tools' account and only logins the wanted one names, more approvals, a push already refused. - * Answers the rule as the forge holds it after, read back. With the administrator's login. */ - async ensureMainRule(owner: string, repo: string, mergers: string[] = mainMergers()): Promise<{ created: boolean; rule: BranchProtection }> { + /** The rule that applies to main in a repository, as its administrator reads it: a rule named main, else the first glob + * rule covering main; null when none does. With the administrator's login. */ + async mainRule(owner: string, repo: string): Promise { const rules = await this.branchProtections(owner, repo); - const own = rules.find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH); - const kept = own?.enable_merge_whitelist === true ? (own.merge_whitelist_usernames ?? []) : []; - const whitelist = kept.length > 0 && kept.includes(mergers[0]) && kept.every((u) => mergers.includes(u)) ? kept : mergers; - const body = { ...mainRuleBody(mergers), merge_whitelist_usernames: whitelist, block_admin_merge_override: true }; - const path = `${repoPath(owner, repo)}/branch_protections`; - if (own) { - const res = await this.request(`${path}/${MAIN_BRANCH}`, { method: "PATCH", body: JSON.stringify(body) }); - if (res.status !== 200) GiteaAdmin.fail(`${path}/${MAIN_BRANCH}`, res); - } else { - const covering = rules.filter((r) => ruleCovers(String(r.rule_name ?? r.branch_name ?? ""), MAIN_BRANCH)); - const approvals = Math.max(0, ...covering.map((r) => Number(r.required_approvals ?? 0))); - const res = await this.request(path, { method: "POST", body: JSON.stringify({ rule_name: MAIN_BRANCH, ...(approvals > 0 ? { required_approvals: approvals } : {}), ...body }) }); - if (res.status !== 201) GiteaAdmin.fail(path, res); - } - const after = (await this.branchProtections(owner, repo)).find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH); - if (!after) throw new Error(`main's rule of ${owner}/${repo} was written, but the forge does not list it`); - // The forge answers a write it did not keep: it drops a whitelist login that cannot write the repository, silently. - const listed = after.merge_whitelist_usernames ?? []; - const wrong: string[] = []; - if (after.enable_push !== false) wrong.push(`push is ${after.enable_push === true ? "allowed" : "not refused"}`); - if (after.enable_merge_whitelist !== true || !listed.includes(mergers[0])) wrong.push(`the merge whitelist [${listed.join(", ")}] leaves out ${mergers[0]}`); - if (after.enable_status_check !== false) wrong.push("a status is still required"); - if (wrong.length > 0) throw new Error(`main's rule of ${owner}/${repo} was written, but the forge holds otherwise: ${wrong.join("; ")}`); - return { created: !own, rule: after }; + return rules.find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH) + ?? rules.find((r) => ruleCovers(String(r.rule_name ?? r.branch_name ?? ""), MAIN_BRANCH)) ?? null; } - private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write" | "admin"): Promise { + /** Put an account on main's merge whitelist in a repository of a user, and nothing else (the operator, 2026-10-26: + * write, the rule untouched). Only an existing rule whose merge whitelist is on is changed, and only its whitelist: the + * account is added to the logins already there. Push, status checks, approvals and the administrator override stay as + * the operator set them; with no rule, or the whitelist off, nothing is written (whoever writes may merge). Read back: + * the forge drops a whitelist login that cannot write the repository and still answers success, so a whitelist read back + * without the account fails loud. Answers the rule as the forge holds it after. With the administrator's login. */ + async ensureMergerListed(owner: string, repo: string, account: string): Promise<{ changed: boolean; rule: BranchProtection | null }> { + const rule = await this.mainRule(owner, repo); + const listed = rule?.merge_whitelist_usernames ?? []; + if (!rule || rule.enable_merge_whitelist !== true || listed.includes(account)) return { changed: false, rule }; + const name = String(rule.rule_name ?? rule.branch_name); + const path = `${repoPath(owner, repo)}/branch_protections/${encodeURIComponent(name)}`; + const res = await this.request(path, { method: "PATCH", body: JSON.stringify({ merge_whitelist_usernames: [...listed, account] }) }); + if (res.status !== 200) GiteaAdmin.fail(path, res); + const after = (await this.branchProtections(owner, repo)).find((r) => (r.rule_name ?? r.branch_name) === name); + if (!after) throw new Error(`main's rule ${name} of ${owner}/${repo} was written, but the forge does not list it`); + const held = after.merge_whitelist_usernames ?? []; + if (after.enable_merge_whitelist === true && !held.includes(account)) { + throw new Error(`${account} was added to the merge whitelist of main's rule ${name} of ${owner}/${repo}, but the forge ` + + `holds [${held.join(", ")}]: it drops a login that cannot write the repository`); + } + return { changed: true, rule: after }; + } + + private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write"): Promise { const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}`; const res = await this.request(path, { method: "PUT", body: JSON.stringify({ permission }) }); if (res.status === 204 || res.status === 200) return; diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 2f1cd6aa..02644863 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -628,9 +628,8 @@ test("the merging account is given write on each repository the indexer's module await writes.take({ key: "car-hunter-2", op: "put", value: { v: 1, repository: "jschoubben/car-hunter" } }); assert.equal(forge.calls.length, from, "a repository already written asks nothing more"); const repo = (full: string) => forgeRepos.find((x) => x.full === full)!; - assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "admin", "a user's repository is administered, not only written"); - assert.deepEqual(userRules.get("jschoubben/car-hunter"), { rule_name: "main", enable_push: false, enable_merge_whitelist: true, - merge_whitelist_usernames: [TOOLS_ACCOUNT], enable_status_check: false, status_check_contexts: [], block_admin_merge_override: true }); + assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "write", "a user's repository is written, not administered"); + assert.equal(userRules.has("jschoubben/car-hunter"), false, "a repository with no rule for main is given none"); assert.equal(repo("novox/hq").collaborators.size, 0, "the organisation's repository is written through its team, no collaborator"); assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, true); assert.deepEqual(said, [{ type: WRITABLE_EVENT, body: { owner: "jschoubben", repo: "car-hunter", account: TOOLS_ACCOUNT } }]); @@ -658,23 +657,28 @@ test("the merging account is given write on each repository the indexer's module assert.ok(!forge.calls.slice(puts).some((c) => c.startsWith("PUT ")), "no write is asked on a repository no module builds from"); assert.equal(repo("jschoubben/dotfiles").collaborators.get(TOOLS_ACCOUNT), "read"); repoFaults.refuse.clear(); - // A stronger rule the operator set (a whitelist of the tools' account alone, two approvals) is kept, never weakened. - userRules.set("jschoubben/snake", { rule_name: "main", enable_push: true, enable_merge_whitelist: true, - merge_whitelist_usernames: [TOOLS_ACCOUNT], required_approvals: 2, block_admin_merge_override: false }); + // The operator's rule is left as set (the operator, 2026-10-26: "Write, rule untouched"): only mesh-tools is added to its + // merge whitelist; a status the mesh does not set and required approvals are said as blocking the mesh's merge, unchanged. + const operators = { rule_name: "main", enable_push: true, enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"], + enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "ci/external"], required_approvals: 2, block_admin_merge_override: false }; + userRules.set("jschoubben/snake", structuredClone(operators)); const after = await grantTool.run({ repositories: ["jschoubben/snake"] }) as any; assert.deepEqual(after.granted, ["jschoubben/snake"]); - assert.deepEqual(after.main_rule.map((x: any) => [x.repository, x.created, x.rule.enable_push, x.rule.required_approvals, x.rule.block_admin_merge_override]), - [["jschoubben/snake", false, false, 2, true]]); - assert.equal(repo("jschoubben/snake").collaborators.get(TOOLS_ACCOUNT), "admin"); - assert.deepEqual(((await unwritableTool.run({})) as any).unwritable, []); - // The forge drops a whitelist login that cannot write and still answers 200: main's rule is read back and that fails loud. - const admin = GiteaAdmin.fromEnv(env); - await assert.rejects(admin.ensureMainRule("jschoubben", "dotfiles"), - /main's rule of jschoubben\/dotfiles was written, but the forge holds otherwise: the merge whitelist \[\] leaves out mesh-tools/); - userRules.get("jschoubben/snake").enable_status_check = true; - userRules.get("jschoubben/snake").enable_push = true; - const held = await admin.ensureMainRule("jschoubben", "snake"); - assert.deepEqual([held.rule.enable_push, held.rule.enable_status_check, held.rule.merge_whitelist_usernames], [false, false, [TOOLS_ACCOUNT]]); + assert.equal(repo("jschoubben/snake").collaborators.get(TOOLS_ACCOUNT), "write"); + assert.deepEqual(userRules.get("jschoubben/snake"), { ...operators, merge_whitelist_usernames: ["jschoubben", TOOLS_ACCOUNT] }, "only the whitelist changed"); + assert.deepEqual(after.merge_whitelist, [{ repository: "jschoubben/snake", changed: true, listed: ["jschoubben", TOOLS_ACCOUNT] }]); + const blockedWhy = "writable, but main's rule blocks the mesh's merge: it requires the status ci/external, which the mesh does not set; " + + "it requires 2 approving reviews on the forge, which the gate does not give"; + assert.deepEqual(after.merge_blocked, [{ repository: "jschoubben/snake", why: blockedWhy }]); + const again = await unwritableTool.run({}) as any; + assert.deepEqual([again.unwritable, again.merge_blocked], [[], [{ repository: "jschoubben/snake", why: blockedWhy }]]); + const writesBefore = forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length; + assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_whitelist[0].changed, false); + assert.equal(forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length, writesBefore, "a listed account is not written again"); + // The forge drops a whitelist login that cannot write and still answers 200: the whitelist is read back and that fails loud. + userRules.set("jschoubben/dotfiles", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"] }); + await assert.rejects(GiteaAdmin.fromEnv(env).ensureMergerListed("jschoubben", "dotfiles", TOOLS_ACCOUNT), + /mesh-tools was added to the merge whitelist of main's rule main of jschoubben\/dotfiles, but the forge holds \[jschoubben\]: it drops a login that cannot write/); userRules.delete("jschoubben/dotfiles"); // A module gone from the indexer leaves its repository out of the delivered list. await writes.take({ key: "snake", op: "delete" }); diff --git a/modules/gitea/writes.ts b/modules/gitea/writes.ts index 70ef948c..5237625c 100644 --- a/modules/gitea/writes.ts +++ b/modules/gitea/writes.ts @@ -18,14 +18,18 @@ // `repo.writable` {owner, repo, account}: mesh-delivery-gate, which does not ask the operator's Approve of a pull request // whose merge the forge would refuse, reads that repository again on it. // -// **A repository of a user is administered, not only written** (the operator, 2026-10-26: "mesh-tools should get all the -// permissions I give him"): the account is an admin collaborator there and puts main's rule right (client.ts ensureMainRule). +// **A repository of a user is written, and main's rule is left as the operator set it** (the operator, 2026-10-26, asked +// "Write, rule untouched"): the account is a write collaborator there, and only added to main's merge whitelist when that +// rule keeps one (client.ts ensureMergerListed). What else in main's rule stops the mesh's merge (a status the mesh does not +// set, approvals the gate does not give) is changed by nobody but said: both tools list such a repository as writable, but +// blocked, with why (merge_blocked). // // Not taken away: a repository whose last module goes from the indexer keeps its collaborator until a hand removes it; // gitea_unwritable_repos and gitea_grant_write say the delivered list as it is now. import type { ForgeRepository } from "./client.js"; import type { BranchProtection } from "./protection.js"; +import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js"; /** The indexer's state this module reads: one key per registered module, its value naming the repository. */ export const INDEXER_MODULES = "mesh-delivery-indexer.modules"; @@ -37,10 +41,33 @@ export interface WriteForge { everyRepository(): Promise; repository(owner: string, repo: string): Promise; repositoryPermission(owner: string, repo: string, user: string): Promise; - addAdminCollaborator(owner: string, repo: string, user: string): Promise; - ensureMainRule(owner: string, repo: string): Promise<{ created: boolean; rule: BranchProtection }>; + addWriteCollaborator(owner: string, repo: string, user: string): Promise; + mainRule(owner: string, repo: string): Promise; + ensureMergerListed(owner: string, repo: string, account: string): Promise<{ changed: boolean; rule: BranchProtection | null }>; } +/** The statuses every pull request the mesh delivers carries (mesh-delivery's checks); a rule requiring another blocks. */ +const MESH_CONTEXTS = new Set([CHECK_CONTEXT, REPO_CHECK_CONTEXT]); + +/** What in main's rule stops the mesh's merge, in words; empty when nothing does. Changed by nobody here: the operator's. */ +export function mergeBlocks(rule: BranchProtection | null, account: string): string[] { + if (!rule) return []; + const blocks: string[] = []; + if (rule.enable_merge_whitelist === true && !(rule.merge_whitelist_usernames ?? []).includes(account)) { + blocks.push(`its merge whitelist [${(rule.merge_whitelist_usernames ?? []).join(", ")}] leaves out ${account} (gitea_grant_write adds it)`); + } + if (rule.enable_status_check === true) { + for (const c of rule.status_check_contexts ?? []) { + if (!MESH_CONTEXTS.has(c)) blocks.push(`it requires the status ${c}, which the mesh does not set`); + } + } + const approvals = Number(rule.required_approvals ?? 0); + if (approvals > 0) blocks.push(`it requires ${approvals} approving review${approvals === 1 ? "" : "s"} on the forge, which the gate does not give`); + return blocks; +} + +const blockedWhy = (blocks: string[]): string => `writable, but main's rule blocks the mesh's merge: ${blocks.join("; ")}`; + /** One change of the indexer's modules state, as the SDK's state watch hands it. */ export interface ModulesChange { key: string; @@ -56,12 +83,13 @@ export interface Written { delivered: string[]; /** Made writable now, each read back from the forge. */ granted: string[]; - /** Administered already (a repository of a user), or written through the organisation's team. */ + /** Written already (a collaborator of a user's repository), or written through the organisation's team. */ already: string[]; - /** Each repository of a user whose main's rule was put right (no push, the merge whitelist, no status, no administrator - * override), read back: what the forge holds for it now. */ - main_rule: { repository: string; created: boolean; rule: { enable_push?: boolean; merge_whitelist_usernames?: string[] | null; - block_admin_merge_override?: boolean; enable_status_check?: boolean; required_approvals?: number } }[]; + /** Each repository of a user: whether the account was added to main's merge whitelist now, and the whitelist read back + * (null: main's rule keeps no whitelist, or there is no rule, and whoever writes may merge). */ + merge_whitelist: { repository: string; changed: boolean; listed: string[] | null }[]; + /** Writable, but main's rule (the operator's, left as set) stops the mesh's merge, and why. */ + merge_blocked: { repository: string; why: string }[]; /** The forge refused, in its own words; or the repository is the organisation's, whose team should write it. */ refused: { repository: string; why: string }[]; /** Repositories the account cannot write that are left so, and why: they build no registered module. */ @@ -69,7 +97,6 @@ export interface Written { } const WRITES = new Set(["write", "admin", "owner"]); -const ADMINS = new Set(["admin", "owner"]); const said = (err: unknown): string => (err instanceof Error ? err.message : String(err)); function split(full: string): [string, string] { @@ -128,7 +155,7 @@ export class DeliveredWrites { "(the watch is not granted or not answered); nothing is granted"); } const delivered = this.delivered(); - const result: Written = { account: this.account, delivered, granted: [], already: [], main_rule: [], refused: [], left_read_only: [] }; + const result: Written = { account: this.account, delivered, granted: [], already: [], merge_whitelist: [], merge_blocked: [], refused: [], left_read_only: [] }; const named = only && only.length ? only.map((x) => String(x).trim()) : null; const targets: string[] = []; for (const full of named ?? delivered) { @@ -162,12 +189,11 @@ export class DeliveredWrites { "set again at the next mint of the tools' token)" }); continue; } - // A repository of a user: the account administers it (the operator, 2026-10-26: "mesh-tools should get all the - // permissions I give him"), so it also puts main's rule right, which only an administrator may. + // A repository of a user: the account writes it, and is put on main's merge whitelist when the rule keeps one. let changed = false; - if (!ADMINS.has(permission)) { + if (!WRITES.has(permission)) { try { - await this.forge.addAdminCollaborator(owner, name, this.account); + await this.forge.addWriteCollaborator(owner, name, this.account); } catch (err) { result.refused.push({ repository: full, why: said(err) }); continue; @@ -179,19 +205,20 @@ export class DeliveredWrites { result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` }); continue; } - if (!ADMINS.has(after)) { + if (!WRITES.has(after)) { result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` }); continue; } changed = true; } try { - const { created, rule } = await this.forge.ensureMainRule(owner, name); - result.main_rule.push({ repository: full, created, rule: { enable_push: rule.enable_push, - merge_whitelist_usernames: rule.merge_whitelist_usernames, block_admin_merge_override: rule.block_admin_merge_override, - enable_status_check: rule.enable_status_check, required_approvals: rule.required_approvals } }); + const { changed: listedNow, rule } = await this.forge.ensureMergerListed(owner, name, this.account); + result.merge_whitelist.push({ repository: full, changed: listedNow, + listed: rule?.enable_merge_whitelist === true ? (rule.merge_whitelist_usernames ?? []) : null }); + const blocks = mergeBlocks(rule, this.account); + if (blocks.length > 0) result.merge_blocked.push({ repository: full, why: blockedWhy(blocks) }); } catch (err) { - result.refused.push({ repository: full, why: `${this.account} administers it, but main's rule could not be put right — ${said(err)}` }); + result.refused.push({ repository: full, why: `${this.account} writes it, but could not be put on main's merge whitelist — ${said(err)}` }); continue; } this.writable.add(full); @@ -221,10 +248,12 @@ export class DeliveredWrites { } /** Every delivered repository the account cannot write, as the forge says now, and why. */ - async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[] }> { + async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[]; + merge_blocked: { repository: string; why: string }[] }> { if (!this.heard) throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read`); const delivered = this.delivered(); const out: { repository: string; permission: string; why: string }[] = []; + const blocked: { repository: string; why: string }[] = []; for (const full of delivered) { const [owner, name] = split(full); let permission: string; @@ -234,12 +263,21 @@ export class DeliveredWrites { out.push({ repository: full, permission: "unknown", why: `the forge did not answer ${this.account}'s permission on it — ${said(err)}` }); continue; } - if (WRITES.has(permission)) continue; + if (WRITES.has(permission)) { + if (owner === this.org) continue; // the organisation's main rules are the mesh's own (mainrules.ts) + try { + const blocks = mergeBlocks(await this.forge.mainRule(owner, name), this.account); + if (blocks.length > 0) blocked.push({ repository: full, why: blockedWhy(blocks) }); + } catch (err) { + blocked.push({ repository: full, why: `writable, but main's rule could not be read — ${said(err)}` }); + } + continue; + } const modules = [...this.modules].filter(([, r]) => r === full).map(([m]) => m).sort(); out.push({ repository: full, permission, why: `builds ${modules.join(", ")}, and the forge gives ${this.account} ` + `"${permission}" on it: its commit status and its merge are refused until gitea_grant_write grants it` }); } - return { account: this.account, delivered: delivered.length, unwritable: out }; + return { account: this.account, delivered: delivered.length, unwritable: out, merge_blocked: blocked }; } } @@ -253,20 +291,23 @@ export function writeTools(writes: () => DeliveredWrites): Tool[] { description: "Every repository that builds a registered module (the indexer's modules, read over the bus) on which " + "the mesh's merging forge account (mesh-tools, which sets commit statuses and merges) cannot write, with the " + "permission the forge gives it there and why: such a pull request can be checked but never merged, and the " + - "delivery gate does not ask its Approve. Empty when it writes every one. Read with the site administrator's login.", + "delivery gate does not ask its Approve. Empty when it writes every one. Its merge_blocked lists each repository of " + + "a user it writes whose main's rule (the operator's, left as set) still stops the mesh's merge: a merge whitelist " + + "without it, a required status the mesh does not set, required approving reviews. Read with the site administrator's login.", input: {}, run: async () => writes().unwritable(), }, { name: "gitea_grant_write", - description: "Give the mesh's merging forge account (mesh-tools) administration, as a collaborator, on every repository " + - "of a user that builds a registered module, and put main's rule right there (no push, the merge whitelist, no status " + - "required, no administrator override; a stronger rule the operator set is kept), or on the repositories named (each " + - "must build one). The answer's main_rule says each rule as the forge holds it after, read back. " + + description: "Give the mesh's merging forge account (mesh-tools) write, as a collaborator, on every repository of a " + + "user that builds a registered module, or on the repositories named (each must build one). Main's rule there is left " + + "as the operator set it; only when it keeps a merge whitelist is mesh-tools added to it, read back (merge_whitelist). " + + "What else in that rule stops the mesh's merge (a required status the mesh does not set, required approving reviews) " + + "is changed by nobody and said in merge_blocked. " + "The module does this itself at its start and when a module of a new repository is registered; this is the same " + "act by hand, such as when a merge or a commit status was refused 403. A repository of the organisation novox " + "is written through its team and needs nothing. Answers what was granted (each read back from the forge), what " + - "was administered already, what the forge refused in its own words, and which other repositories it left read-only " + + "was written already, what the forge refused in its own words, and which other repositories it left read-only " + "and why (they build no registered module). Needs the site administrator's login, used for this grant alone.", input: { repositories: { type: "array", items: { type: "string" }, description: "owner/name of each repository; empty or absent, every delivered one" }, From c584dc57b95e52cfa7dc32bdeff20b0362a81de9 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:37:02 +0100 Subject: [PATCH 04/28] gitea: mesh/check is the mesh's own status and never blocks; signed commits, an outdated branch and a rejected review are said as blocking the mesh's merge (issue 1945, review round 2) --- modules/gitea/test/tools-account.test.ts | 12 ++++++++++-- modules/gitea/writes.ts | 7 +++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 02644863..c69edb77 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -660,7 +660,8 @@ test("the merging account is given write on each repository the indexer's module // The operator's rule is left as set (the operator, 2026-10-26: "Write, rule untouched"): only mesh-tools is added to its // merge whitelist; a status the mesh does not set and required approvals are said as blocking the mesh's merge, unchanged. const operators = { rule_name: "main", enable_push: true, enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"], - enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "ci/external"], required_approvals: 2, block_admin_merge_override: false }; + enable_status_check: true, status_check_contexts: ["mesh/check", "mesh/merge-gate", "ci/external"], required_approvals: 2, + block_admin_merge_override: false, require_signed_commits: true, block_on_outdated_branch: true, block_on_rejected_reviews: true }; userRules.set("jschoubben/snake", structuredClone(operators)); const after = await grantTool.run({ repositories: ["jschoubben/snake"] }) as any; assert.deepEqual(after.granted, ["jschoubben/snake"]); @@ -668,13 +669,20 @@ test("the merging account is given write on each repository the indexer's module assert.deepEqual(userRules.get("jschoubben/snake"), { ...operators, merge_whitelist_usernames: ["jschoubben", TOOLS_ACCOUNT] }, "only the whitelist changed"); assert.deepEqual(after.merge_whitelist, [{ repository: "jschoubben/snake", changed: true, listed: ["jschoubben", TOOLS_ACCOUNT] }]); const blockedWhy = "writable, but main's rule blocks the mesh's merge: it requires the status ci/external, which the mesh does not set; " + - "it requires 2 approving reviews on the forge, which the gate does not give"; + "it requires 2 approving reviews on the forge, which the gate does not give; it requires signed commits, which the mesh does not make; " + + "it refuses a branch behind main, which the mesh does not bring up to date; " + + "it refuses a merge after a review asked for changes, which the gate does not clear"; assert.deepEqual(after.merge_blocked, [{ repository: "jschoubben/snake", why: blockedWhy }]); const again = await unwritableTool.run({}) as any; assert.deepEqual([again.unwritable, again.merge_blocked], [[], [{ repository: "jschoubben/snake", why: blockedWhy }]]); const writesBefore = forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length; assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_whitelist[0].changed, false); assert.equal(forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length, writesBefore, "a listed account is not written again"); + // The mesh's own statuses never block: a rule requiring only them, with mesh-tools listed, is not reported. + userRules.set("jschoubben/snake", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: [TOOLS_ACCOUNT], + enable_status_check: true, status_check_contexts: ["mesh/check", "mesh/merge-gate", "mesh/repo-check"] }); + assert.deepEqual(((await unwritableTool.run({})) as any).merge_blocked, []); + assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_blocked, []); // The forge drops a whitelist login that cannot write and still answers 200: the whitelist is read back and that fails loud. userRules.set("jschoubben/dotfiles", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"] }); await assert.rejects(GiteaAdmin.fromEnv(env).ensureMergerListed("jschoubben", "dotfiles", TOOLS_ACCOUNT), diff --git a/modules/gitea/writes.ts b/modules/gitea/writes.ts index 5237625c..552f7d58 100644 --- a/modules/gitea/writes.ts +++ b/modules/gitea/writes.ts @@ -29,7 +29,7 @@ import type { ForgeRepository } from "./client.js"; import type { BranchProtection } from "./protection.js"; -import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js"; +import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT } from "./pulls.js"; /** The indexer's state this module reads: one key per registered module, its value naming the repository. */ export const INDEXER_MODULES = "mesh-delivery-indexer.modules"; @@ -47,7 +47,7 @@ export interface WriteForge { } /** The statuses every pull request the mesh delivers carries (mesh-delivery's checks); a rule requiring another blocks. */ -const MESH_CONTEXTS = new Set([CHECK_CONTEXT, REPO_CHECK_CONTEXT]); +const MESH_CONTEXTS = new Set([CHECK_STATUS, CHECK_CONTEXT, REPO_CHECK_CONTEXT]); /** What in main's rule stops the mesh's merge, in words; empty when nothing does. Changed by nobody here: the operator's. */ export function mergeBlocks(rule: BranchProtection | null, account: string): string[] { @@ -63,6 +63,9 @@ export function mergeBlocks(rule: BranchProtection | null, account: string): str } const approvals = Number(rule.required_approvals ?? 0); if (approvals > 0) blocks.push(`it requires ${approvals} approving review${approvals === 1 ? "" : "s"} on the forge, which the gate does not give`); + if (rule.require_signed_commits === true) blocks.push("it requires signed commits, which the mesh does not make"); + if (rule.block_on_outdated_branch === true) blocks.push("it refuses a branch behind main, which the mesh does not bring up to date"); + if (rule.block_on_rejected_reviews === true) blocks.push("it refuses a merge after a review asked for changes, which the gate does not clear"); return blocks; } From 3cd826830b6552e4ae9809fb6d94eab5da95537c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:00:54 +0100 Subject: [PATCH 05/28] The build modules' fetch step says which changed files are compiled programs, so the checker can refuse them (issue 1577) --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 12 +++++++----- .../cmd/mesh-build-go/preview.go | 3 +++ .../cmd/mesh-build-go/preview_test.go | 2 +- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 2 ++ .../cmd/mesh-build-image/fetch.go | 19 ++++++++++++------- modules/mesh-build-image/go.mod | 2 +- modules/mesh-build-image/go.sum | 2 ++ .../cmd/mesh-build-node/fetch.go | 12 +++++++----- .../cmd/mesh-build-node/preview.go | 3 +++ modules/mesh-build-node/go.mod | 2 +- modules/mesh-build-node/go.sum | 2 ++ 12 files changed, 42 insertions(+), 21 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index f612dbb5..e9d2f57f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -42,11 +42,13 @@ type FetchResult struct { // head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a // module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the // files, and the workspace is left at the trunk commit. - Base string `json:"base,omitempty"` - Merge string `json:"merge,omitempty"` - Conflicts []string `json:"conflicts,omitempty"` - Changed []string `json:"changed,omitempty"` - Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + // Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge. + Programs *[]buildstep.Program `json:"programs,omitempty"` + Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` // Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which // build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4). Beside []Sibling `json:"beside,omitempty"` diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 067dcfff..1636678e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -41,6 +41,9 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base } res.Base = mr.Base res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if mr.Merge != "" { + res.Programs = &mr.Programs + } if res.Merge == "" { return nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go index 55877c84..6df51b81 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go @@ -77,7 +77,7 @@ func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) { t.Fatal(err) } if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 || - strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" { + strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" || res.Programs == nil || len(*res.Programs) != 0 { // 1577: a merge says it found no program t.Errorf("result = %s", st.Result) } mt, ok := res.Trees["modules/m"] diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 868b22d2..60c998fe 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index c86aa01f..eaa82453 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-image/cmd/mesh-build-image/fetch.go b/modules/mesh-build-image/cmd/mesh-build-image/fetch.go index e07bbfda..b139be23 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/fetch.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/fetch.go @@ -31,13 +31,15 @@ const ForgeAlias = "mesh-forge" // files in conflict, the paths the merge changes and the tree of each module directory). A head that does not // merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is left at the trunk. type FetchResult struct { - Workspace string `json:"workspace"` - Commit string `json:"commit"` - Base string `json:"base,omitempty"` - Merge string `json:"merge,omitempty"` - Conflicts []string `json:"conflicts,omitempty"` - Changed []string `json:"changed,omitempty"` - Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` + Workspace string `json:"workspace"` + Commit string `json:"commit"` + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + // Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge. + Programs *[]buildstep.Program `json:"programs,omitempty"` + Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` } func (m *Module) fetchTool() StepTool { @@ -127,6 +129,9 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base } res.Base = mr.Base res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if mr.Merge != "" { + res.Programs = &mr.Programs + } if res.Merge == "" { return nil } diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 52658707..c384556e 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index ba65afa3..8efd6402 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index 5f712e16..31307a46 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -51,11 +51,13 @@ type FetchResult struct { // commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory. // A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is // left at the trunk commit with nothing installed. - Base string `json:"base,omitempty"` - Merge string `json:"merge,omitempty"` - Conflicts []string `json:"conflicts,omitempty"` - Changed []string `json:"changed,omitempty"` - Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + // Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge. + Programs *[]buildstep.Program `json:"programs,omitempty"` + Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` // Group is the group's package the fetch packed and served to the TypeScript members (the call's `package`): // "@ ". Group string `json:"group,omitempty"` diff --git a/modules/mesh-build-node/cmd/mesh-build-node/preview.go b/modules/mesh-build-node/cmd/mesh-build-node/preview.go index 6d641b9b..48011ed7 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/preview.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/preview.go @@ -27,6 +27,9 @@ func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResul } res.Base = mr.Base res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if mr.Merge != "" { + res.Programs = &mr.Programs + } if res.Merge == "" { return nil } diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index e79f663b..f0e1bfe5 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 28122ac9..b6238d01 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= From 69a3b3f7e0ea4264c9e403963913586af7cbe827 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:03:18 +0100 Subject: [PATCH 06/28] The forge tools cut a request the forge never answers and refuse a file too big for an answer without fetching it: a 5 MB read stalled every read (issue 1577) --- modules/gitea/client.ts | 46 +++++++++++++++++++++++++++---- modules/gitea/test/paging.test.ts | 45 ++++++++++++++++++++++++++++++ modules/gitea/tools/index.ts | 2 +- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index a36e113f..16571f17 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -174,8 +174,27 @@ function ignoreSecretSetting(cfg: Record, key: string): void { console.error(`the setting ${key} is ignored: it comes from the vault`); } +/** How long the forge may take to answer one request before it is cut and said (novox/hq issue 1577: a request for a 5 MB + * file never ended, and every read through the module timed out behind it). */ +export const FORGE_REQUEST_TIMEOUT_MS = 20_000; + +/** The most a tool's answer carries of one file: the bus's message bound is 1 MiB and a file travels as text (issue 1577). */ +export const MAX_FILE_BYTES = 512 * 1024; + +/** One request to the forge, cut at the limit and failing loud naming the request. */ +async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise { + try { + return await fetch(url, { ...init, signal: init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs) }); + } catch (err) { + if (err instanceof Error && err.name === "TimeoutError") throw new Error(`Gitea API ${label}: no answer in ${timeoutMs / 1000} s, the request was cut`); + throw err; + } +} + export class GiteaClient { readonly baseUrl: string; + /** The limit of one request; a test sets it short. */ + requestTimeoutMs = FORGE_REQUEST_TIMEOUT_MS; private readonly tokens: TokenSource; /** A token given as a string is one somebody configured; a source decides for itself (token.ts). */ @@ -230,14 +249,14 @@ export class GiteaClient { private send(path: string, options: RequestInit, token: string): Promise { checkSentPath(path); - return fetch(`${this.baseUrl}/api/v1${path}`, { + return timedFetch(`${this.baseUrl}/api/v1${path}`, { ...options, headers: { "Content-Type": "application/json", Authorization: `token ${token}`, ...(options.headers as Record | undefined), }, - }); + }, `${options.method ?? "GET"} ${path}`, this.requestTimeoutMs); } /** Generic authenticated API call — the escape hatch for endpoints without a dedicated method. @@ -771,6 +790,21 @@ export class GiteaClient { /** One file's contents at a ref (default the repository's default branch), decoded. */ async getFile(owner: string, repo: string, path: string, ref?: string): Promise<{ path: string; ref?: string; sha: string; size: number; content: string }> { const qs = ref ? `?ref=${encodeURIComponent(ref)}` : ""; + // The size first, from the file's directory listing, which carries no content: a file over what an answer can carry is + // refused without its body being fetched (issue 1577). A file the listing does not show is asked for as before. + const clean = path.replace(/^\/+|\/+$/g, ""); + const slash = clean.lastIndexOf("/"); + const dir = slash < 0 ? "" : clean.slice(0, slash); + let listing: any[] | null = null; + try { + listing = await this.request(`${repoPath(owner, repo)}/contents${dir ? "/" + pathSegments("file path", dir) : ""}${qs}`); + } catch (err) { + if (!(err instanceof Error) || !/: 404 /.test(err.message)) throw err; + } + const entry = Array.isArray(listing) ? listing.find((e) => e?.path === clean) : undefined; + if (entry && Number(entry.size) > MAX_FILE_BYTES) { + throw new Error(`the file is ${Number(entry.size)} bytes, over the ${MAX_FILE_BYTES / 1024} KiB a tool answer can carry; read it from a clone`); + } const f = await this.request(`${repoPath(owner, repo)}/contents/${pathSegments("file path", path.replace(/^\/+|\/+$/g, ""))}${qs}`); if (!f || f.type !== "file") throw new Error(`Gitea API: ${path} is not a file`); const content = f.encoding === "base64" ? Buffer.from(String(f.content ?? ""), "base64").toString("utf8") : String(f.content ?? ""); @@ -940,14 +974,14 @@ export class GiteaAdmin { * throwing on it — the caller decides which non-2xx codes are idempotent successes. Only an * unexpected status becomes an error, and only where the caller says so. */ private async request(path: string, options: RequestInit = {}): Promise { - const res = await fetch(`${this.baseUrl}/api/v1${path}`, { + const res = await timedFetch(`${this.baseUrl}/api/v1${path}`, { ...options, headers: { "Content-Type": "application/json", Authorization: this.authorization, ...(options.headers as Record | undefined), }, - }); + }, `${options.method ?? "GET"} ${path}`, FORGE_REQUEST_TIMEOUT_MS); const text = await res.text(); let body: any = null; if (text) { @@ -1192,9 +1226,9 @@ export class GiteaAdmin { * other answer rejects (novox/hq issue 120). */ async holdsTeamMember(org: string, team: string, username: string, password: string): Promise { - const me = await fetch(`${this.baseUrl}/api/v1/user`, { + const me = await timedFetch(`${this.baseUrl}/api/v1/user`, { headers: { Authorization: "Basic " + Buffer.from(`${username}:${password}`).toString("base64") }, - }); + }, "GET /user", FORGE_REQUEST_TIMEOUT_MS); if (me.status === 401 || me.status === 403) return false; if (me.status !== 200) throw new Error(`Gitea GET /user as ${username}: ${me.status}`); const teams = await this.request(`/orgs/${encodeURIComponent(org)}/teams?limit=50`); diff --git a/modules/gitea/test/paging.test.ts b/modules/gitea/test/paging.test.ts index a9f40a32..d6eba14d 100644 --- a/modules/gitea/test/paging.test.ts +++ b/modules/gitea/test/paging.test.ts @@ -90,3 +90,48 @@ test("a pass pages until it reaches back past its floor, and no further", async server.close(); } }); + +// **One forge request never holds the others, and a file too big for an answer is not downloaded** (novox/hq issue 1577): a +// request for a 4.9 MB file never ended and every read through the module timed out behind it. +test("a request the forge never answers is cut at the limit and says which request", async () => { + const { GiteaClient } = await import("../client.ts"); + const server = createServer((req) => { if (!req.url?.includes("fast")) return; }); + await new Promise((r) => server.listen(0, r)); + const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t"); + client.requestTimeoutMs = 150; + try { + const started = Date.now(); + await assert.rejects(client.api("/repos/novox/hq/pulls"), /Gitea API GET \/repos\/novox\/hq\/pulls: no answer in 0\.15 s/); + assert.ok(Date.now() - started < 3000); + } finally { + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); + } +}); + +test("a file over 512 KiB is refused from its directory listing, and its body is never asked for", async () => { + const { GiteaClient } = await import("../client.ts"); + const asked: string[] = []; + const server = createServer((req, res) => { + asked.push(req.url ?? ""); + res.setHeader("Content-Type", "application/json"); + if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin?")) { + return res.end(JSON.stringify([{ path: "bin/big", type: "file", size: 5_000_000 }, { path: "bin/small", type: "file", size: 5 }])); + } + if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin/small")) { + return res.end(JSON.stringify({ type: "file", encoding: "base64", sha: "s", size: 5, content: Buffer.from("hello").toString("base64") })); + } + res.statusCode = 404; + res.end("{}"); + }); + await new Promise((r) => server.listen(0, r)); + const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t"); + try { + await assert.rejects(client.getFile("novox", "hq", "bin/big", "main"), + /the file is 5000000 bytes, over the 512 KiB a tool answer can carry; read it from a clone/); + assert.ok(!asked.some((u) => u.includes("/contents/bin/big")), asked.join("\n")); + assert.equal((await client.getFile("novox", "hq", "bin/small", "main")).content, "hello"); + } finally { + await new Promise((r) => server.close(() => r())); + } +}); diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 8f21cdb1..a0a6c0b7 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -374,7 +374,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini // ---- Contents and branches ---- { name: "gitea_get_file", - description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch).", + description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch). Refuses a file over 512 KiB, which a tool answer cannot carry: read it from a clone.", input: { owner: { type: "string", description: "the repository owner" }, repo: { type: "string", description: "the repository name" }, From 854dbfb3f3f073b235163e0c5d6dc7e9d440deba Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:12:12 +0100 Subject: [PATCH 07/28] mesh-issues: a milestone or commit link moves no issue and keeps no worker current (issue 1781) The seed linked 188 issues at once and every one read as moved at the seed's minute, hiding when each really moved; on a worked issue the link also counted as its worker's activity. These links only place an issue, so last-moved and the worker's activity leave them out. --- .../cmd/mesh-issues/list_progress_test.go | 19 ++++++++++++++++++- .../mesh-issues/cmd/mesh-issues/tracker.go | 18 ++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go b/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go index ef252169..844989f1 100644 --- a/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go @@ -2,6 +2,7 @@ package main import ( "reflect" + "strings" "testing" "time" ) @@ -26,6 +27,22 @@ func TestListSaysWhenEachIssueMovedAndKeepsTheLinkedOnes(t *testing.T) { if _, err := h.Link(LinkAsk{Number: quiet, Kind: LinkDesign, Target: "03-DESIGN/01-to-be/47-delivery-from-commit-to-delivered.md"}); err != nil { t.Fatal(err) } + // Issue 1781: a milestone or a commit link places the issue and moves nothing, nor keeps its worker current. + if _, err := h.milestoneSet("lead@shanks", MilestoneSetAsk{Design: 47, ID: "R6", Outcome: "x"}); err != nil { + t.Fatal(err) + } + h.fixing(t, noted) + worked, _ := h.Get(noted) + clock = clock.Add(time.Hour) + for _, l := range []LinkAsk{{Number: noted, Kind: LinkMilestone, Target: "47/R6"}, + {Number: noted, Kind: LinkCommit, Target: "mesh-catalog@" + strings.Repeat("a", 40)}} { + if _, err := h.link("seed@shanks", l); err != nil { + t.Fatal(err) + } + } + if after, _ := h.Get(noted); !after.Worker.Active.Equal(worked.Worker.Active) { + t.Fatalf("a milestone link kept the worker current: %v, was %v", after.Worker.Active, worked.Worker.Active) + } start := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC) for _, c := range []struct { name string @@ -35,7 +52,7 @@ func TestListSaysWhenEachIssueMovedAndKeepsTheLinkedOnes(t *testing.T) { refuse bool }{ {"numbers say moved", ListAsk{Numbers: []uint64{quiet, noted, linked}}, []uint64{quiet, noted, linked}, - []time.Time{start.Add(2 * time.Hour), start.Add(time.Hour), start.Add(2 * time.Hour)}, false}, + []time.Time{start.Add(2 * time.Hour), start.Add(2 * time.Hour), start.Add(2 * time.Hour)}, false}, {"linked to a decision", ListAsk{Linked: "decision:0493"}, []uint64{linked}, nil, false}, {"linked to a design", ListAsk{Linked: " design:03-DESIGN/01-to-be/47-delivery-from-commit-to-delivered.md"}, []uint64{quiet}, nil, false}, {"linked and numbers together", ListAsk{Linked: "decision:0493", Numbers: []uint64{quiet, noted}}, nil, nil, false}, diff --git a/modules/mesh-issues/cmd/mesh-issues/tracker.go b/modules/mesh-issues/cmd/mesh-issues/tracker.go index 2f06b304..03780639 100644 --- a/modules/mesh-issues/cmd/mesh-issues/tracker.go +++ b/modules/mesh-issues/cmd/mesh-issues/tracker.go @@ -675,7 +675,9 @@ func (t *Tracker) link(by string, a LinkAsk) (*Linked, error) { return fmt.Errorf("issue %d already has the %s link to %s", is.Number, kind, target) } is.Links = append(is.Links[:i:i], is.Links[i+1:]...) - touch(is, by, at) + if !placesOnly(kind) { + touch(is, by, at) + } return nil } } @@ -692,7 +694,9 @@ func (t *Tracker) link(by string, a LinkAsk) (*Linked, error) { return fmt.Errorf("issue %d has %d links, the most one issue keeps", is.Number, MostLinks) } is.Links = append(is.Links, Link{Kind: kind, Target: target, By: by, At: at}) - touch(is, by, at) + if !placesOnly(kind) { + touch(is, by, at) + } return nil }) if err != nil { @@ -768,7 +772,7 @@ type Row struct { } // lastMoved is when an issue last changed, as its own record says: the latest of its opening, its closing, its -// notes, its links, its worker's start and last act, its import and what the delivery last said of each fix. +// notes, its links but those that only place it, its worker's start and last act, its import and what the delivery last said of each fix. func lastMoved(is *Issue) time.Time { at := is.Opened later := func(t time.Time) { @@ -783,7 +787,9 @@ func lastMoved(is *Issue) time.Time { later(n.At) } for _, l := range is.Links { - later(l.At) + if !placesOnly(l.Kind) { + later(l.At) + } } if w := is.Worker; w != nil { later(w.Since) @@ -798,6 +804,10 @@ func lastMoved(is *Issue) time.Time { return at } +// placesOnly says a link of this kind only places the issue (its milestone) or names a commit that fixed it: no work +// on it, so it neither moves the issue nor keeps its worker current (issue 1781: the seed linked 188 issues at once). +func placesOnly(kind string) bool { return kind == LinkMilestone || kind == LinkCommit } + // checkLinked is a list's linked filter, kind:target, said the one way a link of that kind is kept. func checkLinked(s string) (kind, target string, err error) { k, t, ok := strings.Cut(strings.TrimSpace(s), ":") From ee48e5a01e4982ab88206ff98d77044ba3556c32 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:32:00 +0100 Subject: [PATCH 08/28] gitea_get_file refuses a bad path before its size listing reaches the forge; the guard test expects the listing (issue 1577) --- modules/gitea/client.ts | 1 + modules/gitea/test/guard.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 16571f17..a087d1ec 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -793,6 +793,7 @@ export class GiteaClient { // The size first, from the file's directory listing, which carries no content: a file over what an answer can carry is // refused without its body being fetched (issue 1577). A file the listing does not show is asked for as before. const clean = path.replace(/^\/+|\/+$/g, ""); + pathSegments("file path", clean); // a path that is another path or user is refused before anything reaches the forge const slash = clean.lastIndexOf("/"); const dir = slash < 0 ? "" : clean.slice(0, slash); let listing: any[] | null = null; diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 6f1431aa..295de67b 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -640,6 +640,7 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud await tool("gitea_delete_branch").run({ ...repo, branch: "feat/x" }); await tool("gitea_get_file").run({ ...repo, path: "modules/gitea/module.json" }).catch(() => undefined); assert.deepEqual(state.asked, ["DELETE /repos/novox/mesh-catalog/branches/feat/x", + "GET /repos/novox/mesh-catalog/contents/modules/gitea", // the file's size first (issue 1577) "GET /repos/novox/mesh-catalog/contents/modules/gitea/module.json"]); }); From 8530a9730fbcac11e9fb917460ac7e31cafa9e91 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:47:38 +0100 Subject: [PATCH 09/28] The build modules take mesh-sdk's report of programs left in a branch's history (issue 1577) --- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 2 ++ modules/mesh-build-image/go.mod | 2 +- modules/mesh-build-image/go.sum | 2 ++ modules/mesh-build-node/go.mod | 2 +- modules/mesh-build-node/go.sum | 2 ++ 6 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 60c998fe..613a4e54 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index eaa82453..43dd8cc6 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 h1:ozuxlh0fOlq1c3mJ4DLNPgQQ+AIaqd6nWd5CQaEe/r4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index c384556e..ef648e05 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index 8efd6402..d8451089 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 h1:ozuxlh0fOlq1c3mJ4DLNPgQQ+AIaqd6nWd5CQaEe/r4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index f0e1bfe5..d3f88ccc 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index b6238d01..00db2862 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 h1:ozuxlh0fOlq1c3mJ4DLNPgQQ+AIaqd6nWd5CQaEe/r4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= From 851e8782d569dde91ae9ba72a377854ea582949b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:49:39 +0100 Subject: [PATCH 10/28] Nodes compose: the catalogue's placement files and their check, mesh-config reading placement, the vault's own and pair credentials, the forge's commit trees (issue 1126) One commit for the branch feat/1126-nodes-compose of mesh-catalog: its 15 commits squashed so its history does not carry two compiled programs committed by mistake and removed later, mesh-config (5.5 MB) and checks/placement (6.2 MB) (issue 1577). The tree equals the former head 2db56067. The operator allowed this rewrite of the feature branch on 2026-10-26. --- .gitignore | 2 + checks/placement/go.mod | 5 + checks/placement/go.sum | 28 +++ checks/placement/main.go | 80 +++++++ checks/placement/main_test.go | 55 +++++ checks/placement/reads-outside | 1 + modules/gitea/client.ts | 22 +- modules/gitea/test/commit-trees.test.ts | 16 +- modules/gitea/tools/index.ts | 16 +- modules/mesh-config/cmd/mesh-config/main.go | 2 +- .../cmd/mesh-config/pulled_in_test.go | 12 + .../mesh-config/cmd/mesh-config/resolve.go | 5 +- .../mesh-config/cmd/mesh-config/sources.go | 32 +++ modules/mesh-config/module.json | 1 + .../mesh-vault/cmd/mesh-vault/nodekey_test.go | 69 ++++-- modules/mesh-vault/cmd/mesh-vault/ownnode.go | 215 ++++++++++++++++++ .../mesh-vault/cmd/mesh-vault/ownnode_test.go | 92 ++++++++ modules/mesh-vault/cmd/mesh-vault/tools.go | 28 +++ modules/mesh-vault/module.json | 3 +- 19 files changed, 661 insertions(+), 23 deletions(-) create mode 100644 checks/placement/go.mod create mode 100644 checks/placement/go.sum create mode 100644 checks/placement/main.go create mode 100644 checks/placement/main_test.go create mode 100644 checks/placement/reads-outside create mode 100644 modules/mesh-vault/cmd/mesh-vault/ownnode.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/ownnode_test.go diff --git a/.gitignore b/.gitignore index a95e38a3..9552fa38 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,8 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image /checks/paths/paths /modules/mesh-build-go/mesh-build-go /modules/mesh-delivery-gate/mesh-delivery-gate +modules/mesh-config/cmd/mesh-config/mesh-config +checks/placement/placement # Built by their modules' build step (issue 1577), never committed. modules/forticlient/cmd/forticlient-tools/forticlient-tools diff --git a/checks/placement/go.mod b/checks/placement/go.mod new file mode 100644 index 00000000..676b2ed7 --- /dev/null +++ b/checks/placement/go.mod @@ -0,0 +1,5 @@ +module checks/placement + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18 diff --git a/checks/placement/go.sum b/checks/placement/go.sum new file mode 100644 index 00000000..790d7b5e --- /dev/null +++ b/checks/placement/go.sum @@ -0,0 +1,28 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026171759-a431746c0e8c h1:zaSfYTJd03b0pF29QTZ6G5ajfjOXRXjHIpOiGu/niHA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026171759-a431746c0e8c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026172611-5b1e990eb4bb h1:Mo+XdKBGFhUbH1XLSyMdkqIL/JhMIUAucjT+ipZoTfg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026172611-5b1e990eb4bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026173336-9789c8379a74 h1:+LrIwBeG0eCN6xOS9jhzL6ag0ipXHZ1vfCFEJqtT1eI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026173336-9789c8379a74/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026174406-6df0dfe208c6 h1:sIagBHevPfpISBRK2r5EemnhHEGubdfEx/l3K6ujomo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026174406-6df0dfe208c6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026180300-f1d7c7377dc4 h1:rKDfp+IHnv903jcCujf5IdTrWd3mnudpNCfKaKvqp3w= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026180300-f1d7c7377dc4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026181846-91413de33516 h1:xbbss6kQFrAiyy8UEvqh2Ddi2KP+O9oTUSh7GkqmCYU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026181846-91413de33516/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026182519-53c2def1a520 h1:twiziR88X6ttKmw5JTd3aW6xb5bMZZ3Bl7WS2QOP3PE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026182519-53c2def1a520/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18 h1:HWNPfnT9YT9O6xHPLy2y6Q+LYT8XRjvLnZw9O61NETk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/checks/placement/main.go b/checks/placement/main.go new file mode 100644 index 00000000..659edcc6 --- /dev/null +++ b/checks/placement/main.go @@ -0,0 +1,80 @@ +// Command placement is the catalogue's check of its placement files (novox/hq ADR 0549 §1, ADR 0555; issue 1126): every +// `placement/.json` reads as a placement of the node its name says, and the files together are refused, every +// finding named at once, for two modules of one node on one host port, a pin, binding or holding naming a provider not +// placed where it says, a seat held once for the mesh held twice, two nodes with one address on the private network, or +// more than one hub (mesh-sdk delivery.CheckPlacements). +// +// A module the catalogue does not hold and a node not enrolled are the composition check's to refuse: placement names +// modules of every repository the indexer reads and the nodes the mesh enrolled, neither of which this tree holds. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Check reads every placement file under root and answers each finding; none is a catalogue whose placement holds. A +// catalogue with no placement directory has none to check. +func Check(root string) ([]string, error) { + dir := filepath.Join(root, delivery.PlacementDir) + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var found []string + placements := map[string]delivery.Placement{} + for _, e := range entries { + rel := delivery.PlacementDir + "/" + e.Name() + node, ok := delivery.PlacementFileNode(rel) + if e.IsDir() || !ok { + found = append(found, fmt.Sprintf("%s is not a placement file: the directory holds only .json, one per node", rel)) + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, err + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + found = append(found, fmt.Sprintf("%s does not read as a placement: %v", rel, err)) + continue + } + if p.Revision != "" { + found = append(found, fmt.Sprintf("%s names a revision, which only the indexer writes into the key", rel)) + } + placements[node] = p + } + all := func(string) bool { return true } + found = append(found, delivery.CheckPlacements(placements, all, all)...) + sort.Strings(found) + return found, nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: placement ") + os.Exit(2) + } + found, err := Check(os.Args[1]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } +} diff --git a/checks/placement/main_test.go b/checks/placement/main_test.go new file mode 100644 index 00000000..88e45aca --- /dev/null +++ b/checks/placement/main_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The catalogue's placement files as the merge check reads them (ADR 0549 §1, ADR 0555): the real tree holds every +// rule, and a tree that breaks them is refused with every finding named at once. +func TestPlacementFiles(t *testing.T) { + found, err := Check("../..") + if err != nil { + t.Fatal(err) + } + for _, f := range found { + t.Errorf("%s (novox/hq ADR 0549 §1)", f) + } + + root := t.TempDir() + write := func(name, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "placement"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "placement", name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if found, err := Check(root); err != nil || len(found) != 0 { + t.Fatalf("a catalogue with no placement directory: %v, %v", found, err) + } + write("ace.json", `{"v":1,"node":"ace","decided":{"hub":true,"address":"10.77.0.1"},"modules":[{"module":"a","ports":{"80":8080}},{"module":"b","ports":{"81":8080}}]}`) + write("g14.json", `{"v":1,"node":"g14","decided":{"hub":true,"address":"10.77.0.1"},"modules":[{"module":"c","bindings":{"pg":{"node":"ace","module":"pg"}}}]}`) + write("novox.json", `{"v":1,"node":"shanks","modules":[]}`) + write("bad.json", `{"v":1,"node":"bad","modules":{}}`) + write("dated.json", `{"v":1,"node":"dated","revision":"abc","modules":[]}`) + write("notes.md", "x") + found, err = Check(root) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"a and b are both published on host port 8080", "bound to ace/pg", "both have the address 10.77.0.1", + "ace, g14 are each placed as it", "placement file of novox names the node", "bad.json does not read", "dated.json names a revision", + "notes.md is not a placement file"} { + hit := false + for _, f := range found { + hit = hit || strings.Contains(f, want) + } + if !hit { + t.Errorf("no finding says %q; found %v", want, found) + } + } +} diff --git a/checks/placement/reads-outside b/checks/placement/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/placement/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index a36e113f..2a6fe912 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -540,7 +540,7 @@ export class GiteaClient { * **A listing the forge cut is no listing** (it says `truncated` on a page with nothing more to read, or the pages * run past a bound): the call fails naming the repository and the commit, never answering the directories it did * see, since a directory left out would be taken for one without a module. Reads only. */ - async commitTrees(owner: string, repo: string, sha: string, paths?: Record): Promise<{ root: string; trees: Record; paths?: Record> }> { + async commitTrees(owner: string, repo: string, sha: string, paths?: Record, listed?: string[]): Promise<{ root: string; trees: Record; paths?: Record>; listed?: Record> }> { const base = `${repoPath(owner, repo)}/git/trees/${commitSegment(sha)}?recursive=true`; const cut = (why: string) => new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} was cut (${why}); no trees are answered from a part`); const entries: any[] = []; @@ -567,7 +567,23 @@ export class GiteaClient { if (!tree) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} holds ${e.path} but not the tree of ${dir}`); trees[dir] = tree; } - if (paths === undefined) return { root, trees }; + // **Listed directories** (novox/hq ADR 0549 §2, issue 1126): each asked directory's files directly under it, by name + // to blob id, from the same listing: how the forge bridge finds the catalogue's placement files, one per node, at a + // merge. A directory the commit lacks lists nothing (the catalogue before its first placement file). + let files: Record> | undefined; + if (listed !== undefined) { + files = {}; + for (const dir of listed) { + files[dir] = {}; + const prefix = dir === "" ? "" : `${dir}/`; + for (const e of entries) { + if (e?.type !== "blob" || typeof e.path !== "string" || !e.path.startsWith(prefix)) continue; + const name = e.path.slice(prefix.length); + if (name !== "" && !name.includes("/")) files[dir][name] = String(e.sha); + } + } + } + if (paths === undefined) return files === undefined ? { root, trees } : { root, trees, listed: files }; // **Named paths** (novox/hq ADR 0499 §3, issue 1758): each asked path's git object id at the commit, a file's or a // directory's, from the same listing, by directory and by path relative to it. A path the listing lacks fails // naming both, and nothing is answered in part. @@ -582,7 +598,7 @@ export class GiteaClient { named[dir][p] = id; } } - return { root, trees, paths: named }; + return files === undefined ? { root, trees, paths: named } : { root, trees, paths: named, listed: files }; } /** Whether a repository holds a file at a commit: true, false for a 404, thrown when the forge cannot say. */ diff --git a/modules/gitea/test/commit-trees.test.ts b/modules/gitea/test/commit-trees.test.ts index e685a5c0..f1ae6b7e 100644 --- a/modules/gitea/test/commit-trees.test.ts +++ b/modules/gitea/test/commit-trees.test.ts @@ -183,6 +183,15 @@ test("named paths: each asked path's git object id at the commit, a file's or a }); assert.deepEqual(got.trees, { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") }); assert.equal((await client.commitTrees("novox", "hq", COMMIT)).paths, undefined, "no paths asked, none answered"); + // Listed directories (ADR 0549 §2, issue 1126): the files directly under each, not deeper; a missing one lists none. + const listed = await client.commitTrees("novox", "hq", COMMIT, undefined, ["modules/a", "modules/b", "", "placement"]); + assert.deepEqual(listed.listed, { + "modules/a": { "main.go": h("6"), "module.json": h("5") }, + "modules/b": {}, + "": { "README.md": h("1"), "module.json": h("2") }, + placement: {}, + }); + assert.equal((await client.commitTrees("novox", "hq", COMMIT)).listed, undefined, "no directory asked, none listed"); } finally { server.close(); } @@ -209,7 +218,12 @@ test("gitea_commit_trees takes paths as {directory: [paths]} and refuses another const { client, server, asked } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]); try { const tool = getGiteaTools(client).find((t) => t.name === "gitea_commit_trees")!; - assert.ok("paths" in tool.input, "the tool names the argument"); + assert.ok("paths" in tool.input && "listed" in tool.input, "the tool names the arguments"); + const listedAnswer: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, listed: ["modules/d"] } as any); + assert.deepEqual(listedAnswer.listed, { "modules/d": { "notes.md": h("b") } }); + for (const bad of ["modules/d", ["../x"], ["/abs"], [1]]) { + await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: COMMIT, listed: bad } as any), /listed/, JSON.stringify(bad)); + } const got: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths: { "modules/a": ["main.go"] } } as any); assert.deepEqual(got.paths, { "modules/a": { "main.go": h("6") } }); const before = asked.length; diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 8f21cdb1..cfacfce5 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -76,6 +76,17 @@ function namedPaths(raw: unknown): Record | undefined { return out; } +/** The `listed` argument of gitea_commit_trees: [directory], each plain (no `.` or `..` segment, not absolute, the root + * the empty name); anything else is refused before the forge is asked. Absent is absent. */ +function listedDirs(raw: unknown): string[] | undefined { + if (raw === undefined || raw === null) return undefined; + if (!Array.isArray(raw) || raw.some((d) => typeof d !== "string")) throw new Error("listed is [directory, ...]"); + for (const d of raw as string[]) { + if (d !== "" && d.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) throw new Error(`listed: ${JSON.stringify(d)} is not a plain relative directory`); + } + return raw as string[]; +} + export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { return namesChecked(allGiteaTools(gitea, admin)); } @@ -550,19 +561,20 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini }, { name: "gitea_commit_trees", - description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.", + description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. With listed, [directory], the answer also carries listed, {directory: {name: git blob id}} of the files directly under each (the catalogue's placement files, novox/hq ADR 0549 §2). A listing the forge cut is refused, never answered in part. Reads only.", input: { owner: { type: "string", description: "the repository owner" }, repo: { type: "string", description: "the repository name" }, sha: { type: "string", description: "the commit, its full 40-hex sha" }, paths: { type: "object", description: "optional, {directory: [path relative to it]}: the git object id of each named file or directory is answered under paths; the root directory is the empty name" }, + listed: { type: "array", items: { type: "string" }, description: "optional, [directory]: the files directly under each, {directory: {name: git blob id}}, are answered under listed; a directory the commit lacks lists none" }, }, run: async (args) => { const owner = String(args.owner ?? "").trim(), repo = String(args.repo ?? "").trim(); const sha = String(args.sha ?? "").trim(); if (!owner || !repo) throw new Error("name the repository: owner and repo"); if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error("sha is a commit's full 40-hex sha"); - return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths)); + return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths), listedDirs(args.listed)); }, }, { diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 1334b097..63428e01 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -115,7 +115,7 @@ func resolving(c *Config) { } return nil } - for _, read := range []string{AssignmentsRead, DeclaredConfigRead, PredefinedRead} { + for _, read := range []string{AssignmentsRead, PlacementRead, DeclaredConfigRead, PredefinedRead} { go func(read string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { time.Sleep(wait) diff --git a/modules/mesh-config/cmd/mesh-config/pulled_in_test.go b/modules/mesh-config/cmd/mesh-config/pulled_in_test.go index e4120e2c..dbf9ebfb 100644 --- a/modules/mesh-config/cmd/mesh-config/pulled_in_test.go +++ b/modules/mesh-config/cmd/mesh-config/pulled_in_test.go @@ -21,4 +21,16 @@ func TestAModulePulledInHasItsValuesResolvedOnItsNode(t *testing.T) { if r, _ := c.Reconcile(); len(r.Moved) != 4 || !r.Moved[0].Gone { t.Errorf("a module no longer pulled in moved %+v", r.Moved) } + + // 1126 (ADR 0549 §3, §5): a module the catalogue's placement names on a node resolves there as an assigned one does, + // while the two sources stand; it goes when neither names it. + m.mirror(t, PlacementRead, "placement.ace", `{"v":1,"node":"ace","modules":[{"module":"dunst"}]}`) + if r, err := c.Reconcile(); err != nil || len(r.Moved) != 4 { + t.Fatalf("a placed module's values moved %+v (%v)", r.Moved, err) + } + expect(t, m, "ace", "10", SourceDefault) + _ = m.Delete(PlacementRead, "placement.ace") + if r, _ := c.Reconcile(); len(r.Moved) != 4 || !r.Moved[0].Gone { + t.Errorf("a module no longer placed moved %+v", r.Moved) + } } diff --git a/modules/mesh-config/cmd/mesh-config/resolve.go b/modules/mesh-config/cmd/mesh-config/resolve.go index bc21ec61..ccf35335 100644 --- a/modules/mesh-config/cmd/mesh-config/resolve.go +++ b/modules/mesh-config/cmd/mesh-config/resolve.go @@ -20,7 +20,10 @@ const ( PredefinedState = "predefined" ProposalsState = "proposals" - AssignmentsRead = "mesh-controller.assignments" + AssignmentsRead = "mesh-controller.assignments" + // PlacementRead is the indexer's placement, read from the catalogue at each merge (novox/hq ADR 0549 §2, §3): each + // node's key `placement.` with the modules placed there. + PlacementRead = "mesh-delivery-indexer.placement" DeclaredConfigRead = "mesh-controller.declared-config" PredefinedRead = "mesh-controller.predefined" diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index df6292dc..34f8dddf 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -27,6 +27,38 @@ func (c *Config) sources() (Sources, error) { }); err != nil { return s, fmt.Errorf("the assignments cannot be read: %w", err) } + // And every module the catalogue's placement names on a node (ADR 0549 §3): until ADR 0547's third milestone both + // sources stand (ADR 0549 §5), so a pair either names is resolved, and none loses its values while the two differ. + // Placement that cannot be read yet (the indexer's state not on the bus) leaves the assignments alone, said. + standing := map[string]bool{} + for _, a := range s.Assignments { + standing[a.Node+"."+a.Module] = true + } + if err := c.each(PlacementRead, func(key string, raw []byte) error { + node, ok := strings.CutPrefix(key, "placement.") + if !ok { + return nil + } + var p struct { + Node string `json:"node"` + Modules []struct { + Module string `json:"module"` + } `json:"modules"` + } + if err := json.Unmarshal(raw, &p); err != nil || p.Node != node { + problems = append(problems, fmt.Sprintf("the placement %s is not one", key)) + return nil + } + for _, m := range p.Modules { + if m.Module != "" && !standing[node+"."+m.Module] { + standing[node+"."+m.Module] = true + s.Assignments = append(s.Assignments, Assignment{Node: node, Module: m.Module}) + } + } + return nil + }); err != nil { + problems = append(problems, fmt.Sprintf("the catalogue's placement cannot be read, so the assignments alone are resolved: %v", err)) + } if err := c.each(DeclaredConfigRead, func(key string, raw []byte) error { m, err := ParseModuleDeclarations(raw) if err != nil { diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index f93497e1..d6b5bdd0 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -29,6 +29,7 @@ ], "reads": [ "mesh-controller.assignments", + "mesh-delivery-indexer.placement", "mesh-controller.declared-config", "mesh-controller.predefined" ], diff --git a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go index e266dc3d..5a0c738c 100644 --- a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go @@ -5,6 +5,8 @@ package main // key (or a first one) raises `secret-changed` for the machine's core secrets, the same key says nothing. import ( + "git.novox.be/novox/mesh-sdk/go/seal" + "encoding/json" "os" "path/filepath" @@ -100,14 +102,17 @@ authorization { users = [ { user: "novox.mesh-vault", password: "pw", permissions: { publish: { deny: [">"] } - subscribe: { allow: ["mesh.seat.mesh-vault.call.node-key.*"] } + subscribe: { allow: ["mesh.seat.mesh-vault.call.node-key.*", "mesh.seat.mesh-vault.call.own.*"] } allow_responses: true } } { user: "node.ace", password: "pw", permissions: { - publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_ace"] } + publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_ace", "mesh.seat.mesh-vault.call.own.node_ace"] } subscribe: { allow: ["_INBOX.node.ace.>"] } } } { user: "node.g14", password: "pw", permissions: { - publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_g14"] } + publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_g14", "mesh.seat.mesh-vault.call.own.node_g14"] } subscribe: { allow: ["_INBOX.node.g14.>"] } } } + { user: "controller", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.own.controller"] } + subscribe: { allow: ["_INBOX.controller.>"] } } } ] } ` @@ -135,39 +140,47 @@ func TestOnARealBusANodeEngineStatesOnlyItsOwnKey(t *testing.T) { t.Cleanup(s.Shutdown) v := newVault(t, nil) - tool := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key") + tools := map[string]toolDef{"node-key": toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key"), + "own": toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".own")} vaultConn := connectAs(t, s.ClientURL(), "novox.mesh-vault") - if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call.node-key.*", func(m *nats.Msg) { + serve := func(m *nats.Msg) { parts := strings.Split(m.Subject, ".") var args map[string]any _ = json.Unmarshal(m.Data, &args) answer := busAnswer{} - if got, err := tool.runAs(untoken(parts[len(parts)-1]), args); err != nil { + if got, err := tools[parts[len(parts)-2]].runAs(untoken(parts[len(parts)-1]), args); err != nil { answer.Error = err.Error() } else { answer.Result, _ = json.Marshal(got) } raw, _ := json.Marshal(answer) _ = m.Respond(raw) - }); err != nil { - t.Fatal(err) + } + for _, verb := range []string{"node-key", "own"} { + if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call."+verb+".*", serve); err != nil { + t.Fatal(err) + } } _ = vaultConn.Flush() - say := func(user, subject, key string) (string, error) { - raw, _ := json.Marshal(map[string]string{"key": key}) + call := func(user, subject string, args any) (json.RawMessage, error) { + raw, _ := json.Marshal(args) msg, err := connectAs(t, s.ClientURL(), user).Request(subject, raw, 3*time.Second) if err != nil { - return "", err + return nil, err } var a busAnswer _ = json.Unmarshal(msg.Data, &a) if a.Error != "" { - return a.Error, errString(a.Error) + return nil, errString(a.Error) } - return "", nil + return a.Result, nil } - aceKey, _ := aSealingKey(t) + say := func(user, subject, key string) (string, error) { + _, err := call(user, subject, map[string]string{"key": key}) + return "", err + } + aceKey, acePriv := aSealingKey(t) g14Key, _ := aSealingKey(t) if _, err := say("node.ace", "mesh.seat.mesh-vault.call.node-key.node_ace", aceKey); err != nil { t.Fatal(err) @@ -188,4 +201,32 @@ func TestOnARealBusANodeEngineStatesOnlyItsOwnKey(t *testing.T) { if got, held, _ := v.Store.NodeKey("g14"); !held || got != g14Key { t.Errorf("g14's key is %q, %v", got, held) } + + // Issue 1126 (ADR 0547 §2, ADR 0550 §5): `own` on the same bus. node.ace asks its own and gets its module's secret + // sealed to the key it stated, opening with its key alone; g14 cannot ask in ace's name, and asking in its own gets + // none of ace's; the controller is refused, as no node-engine. + if _, err := moveIn(t, v, "node.ace", MoveOf{Node: "ace", Module: "web", Name: "password", Origin: "made", Generation: "g-1", + SealingKey: aceKey}, "ace-web-pw"); err != nil { + t.Fatal(err) + } + raw, err := call("node.ace", "mesh.seat.mesh-vault.call.own.node_ace", map[string]any{}) + var own OwnAnswer + if err != nil || json.Unmarshal(raw, &own) != nil || own.Node != "ace" || len(own.Own) != 1 { + t.Fatalf("ace's own over the bus: %s, %v", raw, err) + } + if value, err := seal.Open(own.Own[0].Sealed, acePriv); err != nil || string(value) != "ace-web-pw" { + t.Errorf("ace's secret does not open with ace's key: %q, %v", value, err) + } + if strings.Contains(string(raw), "ace-web-pw") { + t.Error("ace's secret crossed the bus in the clear") + } + if _, err := call("node.g14", "mesh.seat.mesh-vault.call.own.node_ace", map[string]any{}); err == nil { + t.Error("node.g14 asked ace's own in ace's name") + } + if raw, err := call("node.g14", "mesh.seat.mesh-vault.call.own.node_g14", map[string]any{}); err != nil || strings.Contains(string(raw), "web") { + t.Errorf("g14 asking its own was answered %s, %v", raw, err) + } + if _, err := call("controller", "mesh.seat.mesh-vault.call.own.controller", map[string]any{}); err == nil || !strings.Contains(err.Error(), "not a node-engine") { + t.Errorf("the controller asking own: %v", err) + } } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go new file mode 100644 index 00000000..fb71a9c7 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -0,0 +1,215 @@ +package main + +// A machine's node-engine composes its own modules (novox/hq ADR 0547 §2, issue 1126), and asks the vault for their own +// secrets sealed to its own key (ADR 0349 §11, ADR 0550 §5): the verb `own`, answered to the node-engine of a machine +// alone, about that machine alone (the machine is the caller's, as the bus proved it, never an argument). Each value is +// sealed to the sealing key the machine's engine stated (`node-key`), so what crosses the bus opens on that machine only, +// and the composition carries it sealed (no composition carries a secret in the clear). A module handed to the +// provisioner is left out: its client fills its secrets from `get` (ADR 0349 §17). + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// NodeOwnSecret is one own secret of a module on the asking machine, sealed to the machine's key. +type NodeOwnSecret struct { + Module string `json:"module"` + Name string `json:"name"` + Origin string `json:"origin"` + Generation int `json:"generation"` + Sealed seal.Box `json:"sealed"` +} + +// OwnAnswer is what `own` answers: the machine, the sealing key every value is sealed to (raw X25519, base64, as its +// engine stated it), each own secret of its modules, and each pair credential it asked for that the vault holds. +type OwnAnswer struct { + Node string `json:"node"` + Key string `json:"key"` + Own []NodeOwnSecret `json:"own"` + Credentials []NodeCredential `json:"credentials,omitempty"` + Missing []CredentialAsked `json:"missing,omitempty"` +} + +// CredentialAsked names one pair credential by its ends, as a node composing its own asks for it (mesh-sdk +// compose.CredentialKey): Role is `consumer` when the asking machine consumes it, `provider` when it grants it. +type CredentialAsked struct { + Role string `json:"role"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + ConsumerModule string `json:"consumer_module"` + Provider string `json:"provider"` + Local string `json:"local,omitempty"` +} + +// NodeCredential is one pair credential answered, sealed to the asking machine's key, with the name the vault keeps it +// under (`pair/...` per pair, `provision////` issued for one holder, or the provision's +// one shared value `provision//`). +type NodeCredential struct { + CredentialAsked + Name string `json:"name"` + Generation int `json:"generation"` + Sealed seal.Box `json:"sealed"` +} + +// credentialNames are the names a pair credential may be kept under, the most particular first. +func credentialNames(c CredentialAsked) []string { + pair := "pair/" + c.Provider + "/" + c.Provision + "/" + c.Consumer + "/" + c.ConsumerModule + if c.Local != "" { + pair += "/" + c.Local + } + return []string{pair, "provision/" + c.Provider + "/" + c.Provision + "/" + c.Consumer + "/" + c.ConsumerModule, + "provision/" + c.Provider + "/" + c.Provision} +} + +// credentialRefusal says why node may not have c, or "": a consumer's credential to the machine its consuming module +// stands on, a provider's to the providing machine, and nothing to a machine that is neither. +func (v *Vault) credentialRefusal(node string, c CredentialAsked) string { + for _, part := range []string{c.Provision, c.Consumer, c.ConsumerModule, c.Provider} { + if !namePart.MatchString(part) { + return fmt.Sprintf("%q is not a name the mesh writes", part) + } + } + if c.Local != "" && !namePart.MatchString(c.Local) { + return fmt.Sprintf("%q is not a name the mesh writes", c.Local) + } + switch c.Role { + case "consumer": + if c.Consumer != node { + return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer) + } + if d, ok := v.Askers.(Declared); ok && d.Read != nil { + if raw, err := standing(d.Read)(node + "." + c.ConsumerModule); err == nil && raw == nil { + return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) + } + } + return "" + case "provider": + if c.Provider != node { + return fmt.Sprintf("%s asks as the provider %s, and is not that machine", node, c.Provider) + } + return "" + } + return fmt.Sprintf("the role %q is not consumer or provider", c.Role) +} + +// composeOrigin is the vault's origin of a value as the composition reads it: made by the mesh, or given (accepted at a +// terminal, typed at a desk, issued by a provider's software). +func composeOrigin(origin string) string { + if origin == "" || origin == OriginMade { + return OriginMade + } + return OriginAccepted +} + +// OwnOfNode answers the node-engine of a machine the own secrets of its modules, each sealed to the machine's stated key. +func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, error) { + node, ok := nodeEngineOf(caller) + if !ok { + if caller == "" || caller == stdio.Unproven { + caller = "an unproven caller" + } + return OwnAnswer{}, fmt.Errorf("own answers a machine's node-engine alone, about its own machine; %s is not a node-engine", caller) + } + key, had, err := v.Store.NodeKey(node) + if err != nil { + return OwnAnswer{}, err + } + if !had { + return OwnAnswer{}, fmt.Errorf("the vault holds no secret named own/%s yet: %s's node-engine has stated no sealing key "+ + "(node-key), so there is no key of its own to seal to", node, node) + } + to, err := publicPEM(key) + if err != nil { + return OwnAnswer{}, fmt.Errorf("%s's sealing key as recorded is not usable: %w", node, err) + } + all, err := v.Store.List() + if err != nil { + return OwnAnswer{}, err + } + out := OwnAnswer{Node: node, Key: key} + for _, r := range all { + parts := strings.Split(r.Name, "/") + if len(parts) != 4 || parts[0] != "own" || parts[1] != node { + continue + } + if v.handed(node, parts[2]) { + continue + } + value, err := seal.Open(r.AtRest, v.Own.PrivateKey) + if err != nil { + return OwnAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", r.Name, err) + } + sealed, err := seal.Seal(value, to) + if err != nil { + return OwnAnswer{}, err + } + if err := v.recordHanding(Handing{To: caller, Name: r.Name, Generation: r.Generation, Outcome: "handed"}); err != nil { + return OwnAnswer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", r.Name, err) + } + out.Own = append(out.Own, NodeOwnSecret{Module: parts[2], Name: parts[3], Origin: composeOrigin(r.Origin), + Generation: r.Generation, Sealed: sealed}) + } + sort.Slice(out.Own, func(i, j int) bool { + if out.Own[i].Module != out.Own[j].Module { + return out.Own[i].Module < out.Own[j].Module + } + return out.Own[i].Name < out.Own[j].Name + }) + for _, c := range asked { + if why := v.credentialRefusal(node, c); why != "" { + return OwnAnswer{}, fmt.Errorf("%s may not ask for the credential of %s from %s to %s on %s: %s", caller, + c.Provision, c.Provider, c.ConsumerModule, c.Consumer, why) + } + found := false + for _, name := range credentialNames(c) { + r, held, err := v.Store.Read(name) + if err != nil { + return OwnAnswer{}, err + } + if !held { + continue + } + value, err := seal.Open(r.AtRest, v.Own.PrivateKey) + if err != nil { + return OwnAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", name, err) + } + sealed, err := seal.Seal(value, to) + if err != nil { + return OwnAnswer{}, err + } + if err := v.recordHanding(Handing{To: caller, Name: name, Generation: r.Generation, Outcome: "handed"}); err != nil { + return OwnAnswer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", name, err) + } + out.Credentials = append(out.Credentials, NodeCredential{CredentialAsked: c, Name: name, Generation: r.Generation, Sealed: sealed}) + found = true + break + } + if !found { + out.Missing = append(out.Missing, c) + } + } + v.log("handed %s's node-engine the %d own secrets of its modules and %d credentials (%d not held), sealed to its key", + node, len(out.Own), len(out.Credentials), len(out.Missing)) + return out, nil +} + +// handed says the plan of module on node hands it to the provisioner, so its secrets are filled from `get`, not composed. +// Where the plans cannot be read, nothing counts as handed: the composer leaves out a handed module's secrets itself. +func (v *Vault) handed(node, module string) bool { + d, ok := v.Askers.(Declared) + if !ok || d.Read == nil { + return false + } + raw, err := standing(d.Read)(node + "." + module) + if err != nil || raw == nil { + return false + } + var a plannedSecrets + return json.Unmarshal(raw, &a) == nil && a.Provisioning != nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go new file mode 100644 index 00000000..7693bb36 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// Issue 1126 (ADR 0547 §2, ADR 0550 §5): `own` answers a machine's node-engine the own secrets of its modules, sealed to +// the key that machine's engine stated, about that machine alone; another caller, another machine's secrets and a +// machine with no stated key get nothing. +func TestOwnAnswersAMachineItsModulesSecretsSealedToItsKey(t *testing.T) { + v := newVault(t, nil) + anchorPub, anchorPriv := aSealingKey(t) + laptopPub, _ := aSealingKey(t) + for _, m := range []struct { + caller string + of MoveOf + value string + }{ + {"node.anchor", MoveOf{Node: "anchor", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: anchorPub}, "anchor-web-pw"}, + {"node.anchor", MoveOf{Node: "anchor", Module: "web", Name: "api-key", Origin: "accepted", Generation: "g-1", SealingKey: anchorPub}, "anchor-web-key"}, + {"node.laptop", MoveOf{Node: "laptop", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: laptopPub}, "laptop-web-pw"}, + } { + if _, err := moveIn(t, v, m.caller, m.of, m.value); err != nil { + t.Fatal(err) + } + } + got, err := v.OwnOfNode("node.anchor") + if err != nil { + t.Fatal(err) + } + if got.Node != "anchor" || got.Key != anchorPub || len(got.Own) != 2 { + t.Fatalf("anchor was answered %+v", got) + } + want := map[string][2]string{"api-key": {"accepted", "anchor-web-key"}, "password": {"made", "anchor-web-pw"}} + for _, s := range got.Own { + w, ok := want[s.Name] + if !ok || s.Module != "web" || s.Origin != w[0] || s.Generation < 1 { + t.Errorf("anchor's %s: %+v", s.Name, s) + continue + } + value, err := seal.Open(s.Sealed, anchorPriv) + if err != nil || string(value) != w[1] { + t.Errorf("anchor's %s does not open to its value with anchor's key: %q, %v", s.Name, value, err) + } + } + for caller, says := range map[string]string{ + "controller": "is not a node-engine", + "anchor.web": "is not a node-engine", + "": "an unproven caller", + "node.unknown": "no secret named own/unknown yet", + } { + if _, err := v.OwnOfNode(caller); err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("%q: %v, want a refusal saying %q", caller, err, says) + } + } + if laptop, err := v.OwnOfNode("node.laptop"); err != nil || len(laptop.Own) != 1 || laptop.Own[0].Name != "password" { + t.Errorf("laptop was answered %+v, %v; want its one secret alone", laptop, err) + } + + // A pair credential: laptop's web consumes anchor's postgres. Each end is answered the one value sealed to its own key; + // a credential the vault holds none of is named missing; a machine at neither end is refused. + if _, err := moveIn(t, v, "node.laptop", MoveOf{Provider: "anchor", Provision: "postgres", Consumer: "laptop", + ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: laptopPub}, "pg-pair"); err != nil { + t.Fatal(err) + } + pair := CredentialAsked{Provision: "postgres", Consumer: "laptop", ConsumerModule: "web", Provider: "anchor"} + asConsumer, asProvider := pair, pair + asConsumer.Role, asProvider.Role = "consumer", "provider" + none := CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "laptop", ConsumerModule: "web", Provider: "anchor"} + laptop, err := v.OwnOfNode("node.laptop", asConsumer, none) + if err != nil || len(laptop.Credentials) != 1 || len(laptop.Missing) != 1 || laptop.Missing[0] != none { + t.Fatalf("laptop as consumer: %+v, %v", laptop, err) + } + if !strings.HasPrefix(laptop.Credentials[0].Name, "pair/anchor/postgres/laptop/web") { + t.Errorf("laptop's credential is kept as %s", laptop.Credentials[0].Name) + } + anchor, err := v.OwnOfNode("node.anchor", asProvider) + if err != nil || len(anchor.Credentials) != 1 { + t.Fatalf("anchor as provider: %+v, %v", anchor, err) + } + if value, err := seal.Open(anchor.Credentials[0].Sealed, anchorPriv); err != nil || string(value) != "pg-pair" { + t.Errorf("anchor's copy does not open to the value with its key: %q, %v", value, err) + } + for who, c := range map[string]CredentialAsked{"node.anchor": asConsumer, "node.laptop": asProvider} { + if _, err := v.OwnOfNode(who, c); err == nil || !strings.Contains(err.Error(), "is not that machine") { + t.Errorf("%s asking %+v: %v", who, c, err) + } + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index 1bd6fe9f..2ef64c8d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -251,6 +251,34 @@ func definitions(env *Env) []toolDef { // clause 9, adopt.go): a public half, never the private one. return map[string]any{"recorded": true, "vault_key": v.Own.PublicKey}, nil }}, + {Name: Seat + ".own", + Description: "The own secrets of this machine's modules, each sealed to the machine's sealing key (novox/hq ADR 0547 §2, " + + "ADR 0550 §5, issue 1126): from the node-engine of the machine alone, about its own machine (the bus proves which), " + + "for the compositions it makes itself. Answers the machine, the key each value is sealed to, and per module and " + + "name the origin, the generation and the sealed value; a module handed to the provisioner is left out. With credentials, " + + "also each pair credential the machine consumes or grants, sealed the same way. Never a value in the clear.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "credentials": map[string]any{"type": "array", "description": "the pair credentials the machine's composition " + + "reads, each {role: consumer|provider, provision, consumer, consumer_module, provider, local}: answered " + + "sealed to the machine's key when the machine is that consumer's or that provider, and named missing when " + + "the vault holds none", "items": map[string]any{"type": "object"}}}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + var asked []CredentialAsked + if raw, ok := a["credentials"]; ok && raw != nil { + b, err := json.Marshal(raw) + if err == nil { + err = json.Unmarshal(b, &asked) + } + if err != nil { + return nil, fmt.Errorf("credentials is a list of {role, provision, consumer, consumer_module, provider, local}: %w", err) + } + } + return v.OwnOfNode(caller, asked...) + }}, {Name: Seat + ".move-asks", Description: "Copy rows of the controller's record of asks at a desk into the vault's, as they are and once " + "(design 57's V3): from the controller alone, at most 500 at a time. No value is in any of them.", diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index cdff2835..304b95df 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -177,7 +177,8 @@ "rotate", "move-in", "node-key", - "move-asks" + "move-asks", + "own" ] } ] From 245143d948ea9d28c67dc6e6b4b0fa5de990d39d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:55:31 +0100 Subject: [PATCH 11/28] Build modules run at most as many steps as their node's build CPU gives, and say a check's phases Each of mesh-build-go, -node and -image declares cores-per-step (6, 2, 2) and hands it to the SDK puller, which bounds the configured capacity by floor(slice quota cores / cores-per-step), read live; thread limits and the status tools follow the effective capacity (issue 1806's follow-up: three Go checks on shanks' 12-core slice starved past the 30-minute step bound). mesh-build-go says its lint and test phases with the ask id (issue 1890). --- modules/mesh-build-go/cmd/mesh-build-go/asks.go | 6 +++++- .../mesh-build-go/cmd/mesh-build-go/asks_test.go | 16 +++++++++++++--- modules/mesh-build-go/cmd/mesh-build-go/cpu.go | 8 +++++++- modules/mesh-build-go/cmd/mesh-build-go/lint.go | 4 ++++ modules/mesh-build-go/files/settings.json | 3 ++- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 4 ++-- modules/mesh-build-go/module.json | 8 +++++++- .../cmd/mesh-build-image/asks.go | 6 +++++- .../mesh-build-image/cmd/mesh-build-image/cpu.go | 8 +++++++- modules/mesh-build-image/files/settings.json | 3 ++- modules/mesh-build-image/go.mod | 2 +- modules/mesh-build-image/go.sum | 4 ++-- modules/mesh-build-image/module.json | 8 +++++++- .../mesh-build-node/cmd/mesh-build-node/asks.go | 6 +++++- .../cmd/mesh-build-node/asks_test.go | 16 +++++++++++++--- .../mesh-build-node/cmd/mesh-build-node/cpu.go | 8 +++++++- modules/mesh-build-node/files/settings.json | 3 ++- modules/mesh-build-node/go.mod | 2 +- modules/mesh-build-node/go.sum | 4 ++-- modules/mesh-build-node/module.json | 8 +++++++- 21 files changed, 102 insertions(+), 27 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index f514673c..722d5b48 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -84,9 +84,13 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config // resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own. var watch func(string, func(stdio.ProvisionChange) error) error + // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice + // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + var cores func() float64 if capacity == nil { capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed + cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskGo], m.Say) } ask := m.askSteps(steps) var load func(string) ([]byte, error) @@ -97,7 +101,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep } } return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, Has: m.Has, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, Logf: m.Say, Has: m.Has, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 91783730..02f18241 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "slices" "strings" @@ -125,12 +126,21 @@ func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { raw, _ := os.ReadFile(moduleDir + "/module.json") var doc struct { Config map[string]struct { - Default string `json:"default"` + Default json.RawMessage `json:"default"` } `json:"config"` } _ = json.Unmarshal(raw, &doc) - if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities { - t.Errorf("the capacity's default = %q, want %q", doc.Config[buildstep.CapacitySetting].Default, buildstep.DefaultCapacities) + // Issue 1806's follow-up: the cores a step needs is a setting of its own, in the settings file, with the SDK's default. + if cores := string(doc.Config[buildstep.CoresPerStepSetting].Default); cores != fmt.Sprint(buildstep.DefaultCoresPerStep[delivery.AskGo]) { + t.Errorf("the cores-per-step default = %s, want %g", cores, buildstep.DefaultCoresPerStep[delivery.AskGo]) + } + if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskGo] { + t.Errorf("the settings template holds cores-per-step %g", got) + } + var capacity string + _ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity) + if capacity != buildstep.DefaultCapacities { + t.Errorf("the capacity's default = %q, want %q", capacity, buildstep.DefaultCapacities) } settings := resource(t, m, "settings") if settings["path"] != "${dir:state}/settings.json" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/cpu.go b/modules/mesh-build-go/cmd/mesh-build-go/cpu.go index 24457cb4..c1c00903 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/cpu.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/cpu.go @@ -15,6 +15,8 @@ import ( func (m *Module) cpuLimits() buildcpu.Limits { slots := 1 switch { + case m.Held != nil: + slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice case m.Capacity != nil: slots = m.Capacity() case m.Settings != "": @@ -32,5 +34,9 @@ func (m *Module) threadEnv() []string { // cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime // to (its unit and its slice), read from its cgroup rather than assumed. func (m *Module) cpuStatus() map[string]any { - return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + if m.Held != nil { + out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up) + } + return out } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/lint.go b/modules/mesh-build-go/cmd/mesh-build-go/lint.go index 7fb41f5f..23527db1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/lint.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/lint.go @@ -14,6 +14,7 @@ import ( "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -132,6 +133,9 @@ func (m *Module) lint(ctx context.Context, run forgekey.Runner, tree, d string, // lintAll lints each directory, up to m.parallel() at once like the level's tests; the failures join in directory order // and the error is the first directory's, so the answer does not depend on which ended first (issue 1930). func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string, dirs []string, env func(d string) []string) ([]Failure, error) { + // The ask's phases say it (issue 1890): lint of the directories, then their tests. + buildstep.Phase(ctx, fmt.Sprintf("unit: lint of %d directories", len(dirs))) + defer buildstep.Phase(ctx, fmt.Sprintf("unit: tests of %d directories", len(dirs))) type answer struct { failed []Failure err error diff --git a/modules/mesh-build-go/files/settings.json b/modules/mesh-build-go/files/settings.json index 3175fe39..054af9a2 100644 --- a/modules/mesh-build-go/files/settings.json +++ b/modules/mesh-build-go/files/settings.json @@ -1,3 +1,4 @@ { - "capacity": "shanks=3,ace=2,g14=0,novox=0" + "capacity": "shanks=3,ace=2,g14=0,novox=0", + "cores-per-step": 6 } diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 868b22d2..5e2a7a39 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index c86aa01f..b4a5f723 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 h1:mSBxF9KgIJyNL0gOKYq5naWewUj2yazU/tGOh12Tze0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 2f566cbf..9530f128 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -29,6 +29,12 @@ "description": "How many asks of the Go build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.", "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" + }, + "cores-per-step": { + "type": "number", + "description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", + "minimum": 0.1, + "default": 6 } }, "tools": [ @@ -107,7 +113,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-go", - "takes-config": ["capacity"], + "takes-config": ["capacity", "cores-per-step"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index 45e03207..3775fc7b 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -40,9 +40,13 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config // resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own. var watch func(string, func(stdio.ProvisionChange) error) error + // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice + // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + var cores func() float64 if capacity == nil { capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed + cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskImage], m.Say) } ask := m.askSteps(steps) var load func(string) ([]byte, error) @@ -50,7 +54,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-image/cmd/mesh-build-image/cpu.go b/modules/mesh-build-image/cmd/mesh-build-image/cpu.go index 126b319d..91f1d314 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/cpu.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/cpu.go @@ -15,6 +15,8 @@ import ( func (m *Module) cpuLimits() buildcpu.Limits { slots := 1 switch { + case m.Held != nil: + slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice case m.Capacity != nil: slots = m.Capacity() case m.Settings != "": @@ -32,5 +34,9 @@ func (m *Module) threadEnv() []string { // cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime // to (its unit and its slice), read from its cgroup rather than assumed. func (m *Module) cpuStatus() map[string]any { - return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + if m.Held != nil { + out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up) + } + return out } diff --git a/modules/mesh-build-image/files/settings.json b/modules/mesh-build-image/files/settings.json index 3175fe39..9acf4e20 100644 --- a/modules/mesh-build-image/files/settings.json +++ b/modules/mesh-build-image/files/settings.json @@ -1,3 +1,4 @@ { - "capacity": "shanks=3,ace=2,g14=0,novox=0" + "capacity": "shanks=3,ace=2,g14=0,novox=0", + "cores-per-step": 2 } diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 52658707..0e530690 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index ba65afa3..9e114f39 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 h1:mSBxF9KgIJyNL0gOKYq5naWewUj2yazU/tGOh12Tze0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index ae9c5162..3fe1377f 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -24,6 +24,12 @@ "description": "How many asks of the image build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.", "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" + }, + "cores-per-step": { + "type": "number", + "description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", + "minimum": 0.1, + "default": 2 } }, "tools": [ @@ -78,7 +84,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-image", - "takes-config": ["capacity"], + "takes-config": ["capacity", "cores-per-step"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index 0e1b845d..25bff14f 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -48,9 +48,13 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config // resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own. var watch func(string, func(stdio.ProvisionChange) error) error + // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice + // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + var cores func() float64 if capacity == nil { capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed + cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskNode], m.Say) } ask := m.askSteps(steps) var load func(string) ([]byte, error) @@ -58,7 +62,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskNode, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Prepare(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go index 1985c863..dcecf188 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "strings" "testing" @@ -83,12 +84,21 @@ func TestManifestTakesTheBuildCapacity(t *testing.T) { raw, _ := os.ReadFile(moduleDir + "/module.json") var doc struct { Config map[string]struct { - Default string `json:"default"` + Default json.RawMessage `json:"default"` } `json:"config"` } _ = json.Unmarshal(raw, &doc) - if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities { - t.Errorf("the capacity's default = %q", doc.Config[buildstep.CapacitySetting].Default) + // Issue 1806's follow-up: the cores a step needs is a setting of its own, in the settings file, with the SDK's default. + if cores := string(doc.Config[buildstep.CoresPerStepSetting].Default); cores != fmt.Sprint(buildstep.DefaultCoresPerStep[delivery.AskNode]) { + t.Errorf("the cores-per-step default = %s, want %g", cores, buildstep.DefaultCoresPerStep[delivery.AskNode]) + } + if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskNode] { + t.Errorf("the settings template holds cores-per-step %g", got) + } + var capacity string + _ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity) + if capacity != buildstep.DefaultCapacities { + t.Errorf("the capacity's default = %q", capacity) } if resource(t, m, "settings")["path"] != "${dir:state}/settings.json" { t.Error("no settings file") diff --git a/modules/mesh-build-node/cmd/mesh-build-node/cpu.go b/modules/mesh-build-node/cmd/mesh-build-node/cpu.go index b40ceda4..5936b615 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/cpu.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/cpu.go @@ -15,6 +15,8 @@ import ( func (m *Module) cpuLimits() buildcpu.Limits { slots := 1 switch { + case m.Held != nil: + slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice case m.Capacity != nil: slots = m.Capacity() case m.Settings != "": @@ -32,5 +34,9 @@ func (m *Module) threadEnv() []string { // cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime // to (its unit and its slice), read from its cgroup rather than assumed. func (m *Module) cpuStatus() map[string]any { - return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + if m.Held != nil { + out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up) + } + return out } diff --git a/modules/mesh-build-node/files/settings.json b/modules/mesh-build-node/files/settings.json index 3175fe39..9acf4e20 100644 --- a/modules/mesh-build-node/files/settings.json +++ b/modules/mesh-build-node/files/settings.json @@ -1,3 +1,4 @@ { - "capacity": "shanks=3,ace=2,g14=0,novox=0" + "capacity": "shanks=3,ace=2,g14=0,novox=0", + "cores-per-step": 2 } diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index e79f663b..4f5fd29a 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 28122ac9..8b19ae3a 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 h1:mSBxF9KgIJyNL0gOKYq5naWewUj2yazU/tGOh12Tze0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 701419b2..970cd694 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -25,6 +25,12 @@ "description": "How many asks of the Node build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.", "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" + }, + "cores-per-step": { + "type": "number", + "description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", + "minimum": 0.1, + "default": 2 } }, "tools": [ @@ -100,7 +106,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-node", - "takes-config": ["capacity"], + "takes-config": ["capacity", "cores-per-step"], "merge": "json", "template": "files/settings.json" }, From 1faf317069190d6c21c7ffd3a950af143831fdfe Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:12:21 +0100 Subject: [PATCH 12/28] =?UTF-8?q?Review=20of=20L2:=20the=20vault's=20own?= =?UTF-8?q?=20fails=20closed:=20a=20pair=20credential's=20consumer=20is=20?= =?UTF-8?q?read=20standing,=20the=20shared=20value=20goes=20only=20to=20a?= =?UTF-8?q?=20holder=20its=20record=20names,=20and=20unread=20assignments?= =?UTF-8?q?=20refuse,=20never=20hand=20(C1,=20C2);=20the=20placement=20che?= =?UTF-8?q?ck=20pins=20mesh-sdk=203968849=20(issue=201126,=20ADR=200550=20?= =?UTF-8?q?=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- checks/placement/go.mod | 2 +- checks/placement/go.sum | 2 + .../mesh-vault/cmd/mesh-vault/nodekey_test.go | 1 + modules/mesh-vault/cmd/mesh-vault/ownnode.go | 103 +++++++++++++++--- .../mesh-vault/cmd/mesh-vault/ownnode_test.go | 29 +++++ 5 files changed, 121 insertions(+), 16 deletions(-) diff --git a/checks/placement/go.mod b/checks/placement/go.mod index 676b2ed7..dba4ac0a 100644 --- a/checks/placement/go.mod +++ b/checks/placement/go.mod @@ -2,4 +2,4 @@ module checks/placement go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026190832-3968849b1b36 diff --git a/checks/placement/go.sum b/checks/placement/go.sum index 790d7b5e..b8d29e3f 100644 --- a/checks/placement/go.sum +++ b/checks/placement/go.sum @@ -14,6 +14,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026182519-53c2def1a520 h1:twiziR88 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026182519-53c2def1a520/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18 h1:HWNPfnT9YT9O6xHPLy2y6Q+LYT8XRjvLnZw9O61NETk= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026190832-3968849b1b36 h1:aBkmFUq/zAVdR3PzfgjjC9AxSymNl3eVC7UEZv9tZ0Q= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026190832-3968849b1b36/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= diff --git a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go index 5a0c738c..aca4fc1d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go @@ -140,6 +140,7 @@ func TestOnARealBusANodeEngineStatesOnlyItsOwnKey(t *testing.T) { t.Cleanup(s.Shutdown) v := newVault(t, nil) + v.Askers = Declared{Read: assignments{"ace.web": `{"node":"ace","module":"web"}`}.read, Store: v.Store} tools := map[string]toolDef{"node-key": toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key"), "own": toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".own")} vaultConn := connectAs(t, s.ClientURL(), "novox.mesh-vault") diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go index fb71a9c7..b873dc44 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -9,6 +9,7 @@ package main import ( "encoding/json" + "errors" "fmt" "sort" "strings" @@ -83,10 +84,17 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked) string { if c.Consumer != node { return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer) } - if d, ok := v.Askers.(Declared); ok && d.Read != nil { - if raw, err := standing(d.Read)(node + "." + c.ConsumerModule); err == nil && raw == nil { - return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) - } + // The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses). + read, why := v.assignmentReader(node) + if why != "" { + return why + } + raw, err := read(node + "." + c.ConsumerModule) + if err != nil { + return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) + } + if raw == nil { + return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) } return "" case "provider": @@ -98,6 +106,47 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked) string { return fmt.Sprintf("the role %q is not consumer or provider", c.Role) } +// assignmentReader is the vault's reader of the assignments, or why it has none: the rule `own` keeps reads them, +// and without them it refuses rather than hands (a rule that cannot be read is no rule). +func (v *Vault) assignmentReader(node string) (AssignmentReader, string) { + d, ok := v.Askers.(Declared) + if !ok || d.Read == nil { + return nil, "the vault cannot read the assignments, so it cannot tell what " + node + "'s modules are; nothing is handed" + } + return standing(d.Read), "" +} + +// sharedRefusal says why node may not have the provision's one shared value `provision//` for c, +// or "": the vault's record of who holds it names the consumer's module among its consumers, or the machine as its +// provider. Without that record nothing is handed: the shared value is not one any machine may have by naming a +// provider (ADR 0550 §5). +func (v *Vault) sharedRefusal(node string, c CredentialAsked) (string, error) { + name := "provision/" + c.Provider + "/" + c.Provision + sh, ok, err := v.Store.ReadShared(name) + if err != nil { + return "", err + } + if !ok { + return fmt.Sprintf("the vault keeps no record of who holds %s", name), nil + } + switch c.Role { + case "consumer": + want := consumerOf(c.Consumer, c.ConsumerModule, c.Local) + for _, held := range sh.Consumers { + if held == want { + return "", nil + } + } + return fmt.Sprintf("%s is held by %s, not by %s", name, strings.Join(sh.Consumers, ", "), want), nil + case "provider": + if sh.Provider == node { + return "", nil + } + return fmt.Sprintf("%s is provided from %s, not %s", name, sh.Provider, node), nil + } + return fmt.Sprintf("the role %q is not consumer or provider", c.Role), nil +} + // composeOrigin is the vault's origin of a value as the composition reads it: made by the mesh, or given (accepted at a // terminal, typed at a desk, issued by a provider's software). func composeOrigin(origin string) string { @@ -138,7 +187,11 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e if len(parts) != 4 || parts[0] != "own" || parts[1] != node { continue } - if v.handed(node, parts[2]) { + handed, err := v.handed(node, parts[2]) + if err != nil { + return OwnAnswer{}, err + } + if handed { continue } value, err := seal.Open(r.AtRest, v.Own.PrivateKey) @@ -167,7 +220,8 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e c.Provision, c.Provider, c.ConsumerModule, c.Consumer, why) } found := false - for _, name := range credentialNames(c) { + names := credentialNames(c) + for i, name := range names { r, held, err := v.Store.Read(name) if err != nil { return OwnAnswer{}, err @@ -175,6 +229,17 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e if !held { continue } + if i == len(names)-1 { + // The provision's one shared value: handed only to a machine the record of its holders names. + why, err := v.sharedRefusal(node, c) + if err != nil { + return OwnAnswer{}, err + } + if why != "" { + v.log("%s is not handed %s: %s", node, name, why) + continue + } + } value, err := seal.Open(r.AtRest, v.Own.PrivateKey) if err != nil { return OwnAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", name, err) @@ -200,16 +265,24 @@ func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, e } // handed says the plan of module on node hands it to the provisioner, so its secrets are filled from `get`, not composed. -// Where the plans cannot be read, nothing counts as handed: the composer leaves out a handed module's secrets itself. -func (v *Vault) handed(node, module string) bool { - d, ok := v.Askers.(Declared) - if !ok || d.Read == nil { - return false +// A plan that cannot be read is an error, never "not handed": `own` refuses rather than hands a secret the provisioner +// holds. +func (v *Vault) handed(node, module string) (bool, error) { + read, why := v.assignmentReader(node) + if why != "" { + return false, errors.New(why) } - raw, err := standing(d.Read)(node + "." + module) - if err != nil || raw == nil { - return false + raw, err := read(node + "." + module) + if err != nil { + return false, fmt.Errorf(notYetPrefix+"the assignment %s.%s could not be read, so whether it is handed to the "+ + "provisioner is not known: %w", node, module, err) + } + if raw == nil { + return false, nil } var a plannedSecrets - return json.Unmarshal(raw, &a) == nil && a.Provisioning != nil + if err := json.Unmarshal(raw, &a); err != nil { + return false, fmt.Errorf("the assignment %s.%s is not the controller's: %w", node, module, err) + } + return a.Provisioning != nil, nil } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go index 7693bb36..131e4455 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go @@ -1,6 +1,7 @@ package main import ( + "errors" "strings" "testing" @@ -11,7 +12,9 @@ import ( // the key that machine's engine stated, about that machine alone; another caller, another machine's secrets and a // machine with no stated key get nothing. func TestOwnAnswersAMachineItsModulesSecretsSealedToItsKey(t *testing.T) { + plans := assignments{"anchor.web": `{"node":"anchor","module":"web"}`, "laptop.web": `{"node":"laptop","module":"web"}`} v := newVault(t, nil) + v.Askers = Declared{Read: plans.read, Store: v.Store} anchorPub, anchorPriv := aSealingKey(t) laptopPub, _ := aSealingKey(t) for _, m := range []struct { @@ -89,4 +92,30 @@ func TestOwnAnswersAMachineItsModulesSecretsSealedToItsKey(t *testing.T) { t.Errorf("%s asking %+v: %v", who, c, err) } } + + // Review of L2 (C1, C2): `own` fails closed. A provision's one shared value is handed only to a consumer the record + // of its holders names, never to a machine that names its provider; assignments that cannot be read, or a vault + // that reads none, refuse rather than hand. + if _, err := v.keep("provision/anchor/redis", "shared-redis", "test", "", OriginMade); err != nil { + t.Fatal(err) + } + if err := v.Store.WriteShared(Shared{Name: "provision/anchor/redis", Provider: "anchor", Provision: "redis", + Consumers: []string{"anchor/web"}}); err != nil { + t.Fatal(err) + } + if got, err := v.OwnOfNode("node.laptop", none); err != nil || len(got.Credentials) != 0 || len(got.Missing) != 1 { + t.Errorf("laptop, whose web the shared value's record does not name, got %+v, %v", got, err) + } + anchorRedis := CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"} + if got, err := v.OwnOfNode("node.anchor", anchorRedis); err != nil || len(got.Credentials) != 1 { + t.Errorf("anchor's web, a holder the record names, got %+v, %v", got, err) + } + v.Askers = Declared{Read: func(string) ([]byte, error) { return nil, errors.New("the bucket did not answer") }, Store: v.Store} + if _, err := v.OwnOfNode("node.anchor"); err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Errorf("own with the assignments unread: %v", err) + } + v.Askers = nil + if _, err := v.OwnOfNode("node.anchor", anchorRedis); err == nil || !strings.Contains(err.Error(), "cannot read the assignments") { + t.Errorf("own with no reader of the assignments: %v", err) + } } From 1e6b1578656218c5802b220bc3703d3789a71f94 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:35:47 +0100 Subject: [PATCH 13/28] Pin mesh-sdk feat/1126-nodes-compose 03dce12, the group's head: the checker refuses a pin that is neither trunk nor the group's head (issue 1126) --- checks/placement/go.mod | 2 +- checks/placement/go.sum | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/checks/placement/go.mod b/checks/placement/go.mod index dba4ac0a..29fdcc2b 100644 --- a/checks/placement/go.mod +++ b/checks/placement/go.mod @@ -2,4 +2,4 @@ module checks/placement go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026190832-3968849b1b36 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 diff --git a/checks/placement/go.sum b/checks/placement/go.sum index b8d29e3f..5c5cbdbe 100644 --- a/checks/placement/go.sum +++ b/checks/placement/go.sum @@ -1,21 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026171759-a431746c0e8c h1:zaSfYTJd03b0pF29QTZ6G5ajfjOXRXjHIpOiGu/niHA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026171759-a431746c0e8c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026172611-5b1e990eb4bb h1:Mo+XdKBGFhUbH1XLSyMdkqIL/JhMIUAucjT+ipZoTfg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026172611-5b1e990eb4bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026173336-9789c8379a74 h1:+LrIwBeG0eCN6xOS9jhzL6ag0ipXHZ1vfCFEJqtT1eI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026173336-9789c8379a74/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026174406-6df0dfe208c6 h1:sIagBHevPfpISBRK2r5EemnhHEGubdfEx/l3K6ujomo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026174406-6df0dfe208c6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026180300-f1d7c7377dc4 h1:rKDfp+IHnv903jcCujf5IdTrWd3mnudpNCfKaKvqp3w= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026180300-f1d7c7377dc4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026181846-91413de33516 h1:xbbss6kQFrAiyy8UEvqh2Ddi2KP+O9oTUSh7GkqmCYU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026181846-91413de33516/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026182519-53c2def1a520 h1:twiziR88X6ttKmw5JTd3aW6xb5bMZZ3Bl7WS2QOP3PE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026182519-53c2def1a520/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18 h1:HWNPfnT9YT9O6xHPLy2y6Q+LYT8XRjvLnZw9O61NETk= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026183837-9713c3023a18/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026190832-3968849b1b36 h1:aBkmFUq/zAVdR3PzfgjjC9AxSymNl3eVC7UEZv9tZ0Q= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026190832-3968849b1b36/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 h1:RInAboHBk5THYf6Agvy05iO9p2XV5zGb0ilYa+iSO7o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= From ff2632356b335b1bb1645dc20e588cdbb49a202d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:02:04 +0100 Subject: [PATCH 14/28] The gate says whether a merged delivery is on the machines, derived from the view's reach, and a person settles a merge held without a passing check on the operator's answer (issue 1280, ADR 0556 clauses 4 and 5) --- .../cmd/mesh-delivery-gate/approval.go | 9 +- .../cmd/mesh-delivery-gate/holder.go | 6 +- .../cmd/mesh-delivery-gate/main.go | 50 +++- .../cmd/mesh-delivery-gate/manifest_test.go | 10 +- .../cmd/mesh-delivery-gate/model.go | 4 + .../cmd/mesh-delivery-gate/reached.go | 88 +++++++ .../cmd/mesh-delivery-gate/reached_test.go | 65 ++++++ .../cmd/mesh-delivery-gate/settle.go | 215 ++++++++++++++++++ .../cmd/mesh-delivery-gate/settle_test.go | 87 +++++++ .../cmd/mesh-delivery-gate/table.go | 24 ++ .../cmd/mesh-delivery-gate/table_test.go | 14 +- .../cmd/mesh-delivery-gate/verbs.go | 14 +- modules/mesh-delivery-gate/module.json | 6 +- 13 files changed, 573 insertions(+), 19 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 3c385094..d835ffb4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -579,7 +579,7 @@ func (h *Holder) hearAsks() { return } h.mu.Lock() - keys := h.pendingAsks() + keys := append(h.pendingAsks(), h.settleKeys()...) h.mu.Unlock() for _, key := range keys { answer, err := h.Operator.Where(key) @@ -591,7 +591,12 @@ func (h *Holder) hearAsks() { continue } h.mu.Lock() - said := h.answered(key, *answer) + var said []func() + if h.isSettleKey(key) { + h.settled(key, *answer) // a person's settle ask (settle.go, novox/hq ADR 0556 clause 5) + } else { + said = h.answered(key, *answer) + } h.mu.Unlock() for _, s := range said { s() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 138c7589..9d690822 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -89,6 +89,9 @@ type Holder struct { Turns TurnStore turnsHeld map[string]sdk.Turn turnsLoaded bool + // reach is the delivery view's reach of each repository, by its key (reached.go, novox/hq ADR 0556): what `deliveries` + // and `show` derive whether a merge is on the machines from. Nothing here records it. + reach map[string]sdk.Reach // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). tried owedTries // forgeWanted is whether the forge's open pull requests are to be read at the next React: a check may have @@ -139,6 +142,7 @@ func (h *Holder) init() { h.unchecked = map[string]unchecked{} h.checker = map[string]checkerVerdict{} h.runningSince, h.turnsHeld = map[string]time.Time{}, map[string]sdk.Turn{} + h.reach = map[string]sdk.Reach{} h.started = h.Now() } @@ -757,7 +761,7 @@ func (h *Holder) pullMerged(p PullEvent, late bool) { } ts = append(ts, settle(d, h.factsAt(now))...) if d.State == Held && d.HeldWhy == "" { - d.HeldWhy = "merged without a passing check: only a person decides what follows" + d.HeldWhy = HeldUnchecked } h.moved(d, ts) h.closeGroupOf(d) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index be04c7f1..52d68da4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -2,8 +2,9 @@ // runtime launches. It owns the delivery — one commit in one repository, from its pull request's head to // every machine — and the delivery group above it: their states in one compiled table, kept in its own state // on the bus, every transition said as an event, noted on the commit and shown on the pull request through -// the forge's holder. It asks the Approve of a merge and merges once; it asks the controller nothing and derives nothing -// after the merge (ADR 0539 §17). stdout +// the forge's holder. It asks the Approve of a merge and merges once; it asks the controller nothing and records nothing +// after the merge (ADR 0539 §17): whether a merge is on the machines it derives when asked from the delivery view's reach +// (ADR 0556 clause 4), and a merge held without a passing check is settled on the operator's answer (clause 5). stdout // is the MCP channel; what this module says, it says on stderr. package main @@ -116,6 +117,21 @@ func run(h *Holder, l *listening) { time.Sleep(wait) } }() + // The delivery view's reach (novox/hq ADR 0556 clause 4): how far each repository reached, which `deliveries` and `show` + // derive whether a merge is on the machines from. Each change is kept; nothing is moved by it. + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.State(reachState).Watch("", func(c stdio.StateChange) error { + h.ReachChanged(c.Key, c.Value, c.Op == "delete") + return nil + }) + if err == nil { + return + } + logf("[mesh-delivery] not hearing the delivery view's reach yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } + }() // The checker's verdicts (issue 1302, ADR 0447 §2): the Approve on mesh/check is asked only on its verdict at exactly the // heads an ask covers, and a group's verdict is that one; each change asks for a look. go func() { @@ -290,7 +306,8 @@ func tools(h *Holder, l *listening) []stdio.Tool { list := []stdio.Tool{ {Name: seat + "deliveries", Description: "Every delivery not final, those that ended in the last day, and any stop `stalled` still lists, one line each: its id " + - "(owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, " + + "(owner/repository@commit), its state, its group, what it waits for and since when, and for a merged one whether it is on " + + "the machines (machines, derived from the delivery view's reach, novox/hq ADR 0556). Narrowed by state, " + "repository or group.", Input: map[string]any{"type": "object", "properties": map[string]any{"state": str("one state"), "repository": str("owner/repository"), "group": str("a group's id (its branch name)"), @@ -301,7 +318,8 @@ func tools(h *Holder, l *listening) []stdio.Tool { }}, {Name: seat + "show", Description: "One delivery or group whole: every transition with when and why, and its group. " + - "What the merge moved is source.main; whether it reached the machines is the view's.", + "For a merged delivery, whether its merge is on the machines (machines), derived from the delivery view's reach of its " + + "repository (novox/hq ADR 0556), and else what holds it.", Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("a delivery's id, or a group's")}, "required": []string{"id"}}, Run: func(a map[string]any) (any, error) { return h.Show(strArg(a, "id")) }}, @@ -362,6 +380,30 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return h.Close(strArg(a, "id"), strArg(a, "why")) }}, + {Name: seat + "settle", + Description: "Ask the operator to settle deliveries held because their merge reached the trunk without a passing check: " + + "one ask on the operator's channel for all of them, as an Approve is asked. The verb moves nothing; on the operator's " + + "answer each goes to merged, and whether it is on the machines is derived as for any merge (novox/hq ADR 0556). " + + "Refused, nothing asked, when any delivery named is not held or is held for another reason, naming why.", + Input: map[string]any{"type": "object", "additionalProperties": false, "required": []string{"ids", "why"}, + "properties": map[string]any{ + "ids": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "minItems": 1, + "description": "the held deliveries' ids (owner/repository@commit)"}, + "why": str("why they may be settled, shown to the operator")}}, + Run: func(a map[string]any) (any, error) { + var ids []string + switch v := a["ids"].(type) { + case []any: + for _, x := range v { + if s, ok := x.(string); ok { + ids = append(ids, s) + } + } + case []string: + ids = v + } + return h.Settle(ids, strArg(a, "why"), aPerson) + }}, {Name: "delivery_turns", Description: "Every merge turn (novox/hq ADR 0548 §3): per repository, the approved merges that wait to merge into it in the order of " + "their Approve, the check of another delivery each waits for and since when, whether the gate wrote the turn to its bucket " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 4ca96530..4e16ea29 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -70,8 +70,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if len(m.Seats) != 0 { t.Fatal("the manifest declares a seat; mesh-delivery is the mesh's own") } - // The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans. - if want := []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { + // The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans; and + // settle, a person's ask to settle merges held without a passing check (ADR 0556 clause 5). + if want := []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close", "settle"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want) } // R6-C1: no controller verb is invoked and no plan event is consumed. @@ -98,8 +99,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { t.Fatalf("state %v", buckets) } // What it reads: the teller's answers and the checker's verdicts (issue 1302). The carry from today's module's buckets, the - // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). - if !reflect.DeepEqual(m.Reads, []string{"mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts"}) { + // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). The delivery view's reach, which + // `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4); no forge tool is asked for it. + if !reflect.DeepEqual(m.Reads, []string{"mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts", "mesh-delivery-view.reach"}) { t.Fatalf("reads %v", m.Reads) } // It calls no controller verb and reads the forge with three tools; the forge's writes are the bridge's (ADR 0437 §8). It diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 9f2a4de8..e46c968f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -105,6 +105,10 @@ type Delivery struct { // trunk's head, before the operator is asked (novox/hq ADR 0421 §3, issue 1022). Cleared when it is asked. FreshWanted bool `json:"fresh_wanted,omitempty"` + // Settle is the settle ask a person had written for it while it is held for a merge without a passing check (settle.go, + // novox/hq ADR 0556 clause 5): its key in the gate's asks, why, who asked and when, and the operator's answer once read. + Settle *SettleAsk `json:"settle,omitempty"` + Transitions []Transition `json:"transitions,omitempty"` // Owed are the forge's and the bus's effects of its transitions not yet done: said only after the // delivery is kept, retried until done. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go new file mode 100644 index 00000000..5bf5fb37 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "strings" + + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Whether a merge is on the machines (novox/hq ADR 0556 clause 4). The gate records only the merge (ADR 0539 §17, ADR 0429 +// §1): it reads the delivery view's reach of each repository, which names the repository's reached head (the newest merge every +// node runs a build containing), and derives when asked, for a merged delivery, whether its merge is at or before that head. +// The order of two merges into one trunk is the gate's own: the moment each was merged, the trunk being a line of merges. A +// reached head the gate holds no merge of is said, never asked of the forge. + +// reachState is the delivery view's state the gate reads. +const reachState = sdk.ViewModule + ".reach" + +// ReachChanged takes one change of the view's reach. +func (h *Holder) ReachChanged(key string, value []byte, deleted bool) { + h.mu.Lock() + defer h.mu.Unlock() + if deleted { + delete(h.reach, key) + return + } + r, err := sdk.DecodeReach(value) + if err != nil { + h.Logf("[mesh-delivery] the view's reach %s is not readable, so its merges read as not known: %v", key, err) + delete(h.reach, key) + return + } + h.reach[key] = r +} + +// The words a merged delivery's machines are said in. +const ( + onMachines = "on the machines" + notYetOn = "not yet on the machines: " + notKnownWhere = "not known: " +) + +// machines says whether a merged delivery's merge is on the machines, derived from the view's reach; empty for a delivery +// that is not merged. Under the lock. +func (h *Holder) machines(d *Delivery) string { + if d.State != Merged || d.MergedAs == "" { + return "" + } + key, err := sdk.RepositoryKey(d.Repository) + if err != nil { + return notKnownWhere + err.Error() + } + r, ok := h.reach[key] + switch { + case !ok: + return notKnownWhere + "the delivery view says no reach of " + d.Repository + case r.Reached == "": + return notYetOn + holdsSaid(r) + case sameCommit(d.MergedAs, r.Reached): + return onMachines + } + head := h.mergeOf(d.Repository, r.Reached) + switch { + case head == nil: + return notKnownWhere + fmt.Sprintf("the reached head %s of %s is no merge this gate holds", shortOf(r.Reached, 12), d.Repository) + case !d.MergedAt.After(head.MergedAt): + return onMachines + } + return notYetOn + holdsSaid(r) +} + +// holdsSaid is what holds a repository's next head, in words. +func holdsSaid(r sdk.Reach) string { + if len(r.Holds) == 0 { + return "the delivery view names nothing that holds it yet" + } + return strings.Join(r.Holds, "; ") +} + +// mergeOf is the delivery of a repository that merged as the commit; nil when none does. Under the lock. +func (h *Holder) mergeOf(repository, commit string) *Delivery { + for _, d := range h.deliveries { + if d.MergedAs != "" && strings.EqualFold(d.Repository, repository) && sameCommit(d.MergedAs, commit) { + return d + } + } + return nil +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go new file mode 100644 index 00000000..89493406 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "strings" + "testing" + "time" + + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1280 (novox/hq ADR 0556 clause 4): `deliveries` and `show` say for a merged delivery whether it is on the machines, derived +// when asked from the view's reach and the gate's own order of merges: at or before the reached head it is, after it not yet +// and why; a repository the view has no reach of, or a reached head the gate holds no merge of, is not known. Nothing records it. +func TestAMergedDeliverySaysWhetherItIsOnTheMachines(t *testing.T) { + w := newWorld(t) + at := w.now.Add(-3 * time.Hour) + merge := func(commit string, n int) *Delivery { + d := &Delivery{ID: "novox/web@" + commit[:12], Repository: "novox/web", Commit: commit[:12], Number: n, State: Merged, + Since: at, MergedAs: strings.Repeat(commit[:1], 40), MergedAt: at.Add(time.Duration(n) * time.Minute)} + w.h.mu.Lock() + w.h.keep(d) + w.h.mu.Unlock() + return d + } + first, second, third := merge("111111111111", 1), merge("222222222222", 2), merge("333333333333", 3) + reach := func(r sdk.Reach) { + key, _ := sdk.RepositoryKey(r.Repository) + raw, err := sdk.EncodeReach(r) + if err != nil { + t.Fatal(err) + } + w.h.ReachChanged(key, raw, false) + } + machines := func(d *Delivery) string { w.h.mu.Lock(); defer w.h.mu.Unlock(); return w.h.machines(d) } + if got := machines(first); !strings.HasPrefix(got, notKnownWhere) { + t.Errorf("with no reach: %q", got) + } + reach(sdk.Reach{Repository: "novox/web", Reached: second.MergedAs, ReachedSrc: 7, Holds: []string{"web on g14 runs the build of 2222, not yet that of 3333"}}) + for d, want := range map[*Delivery]string{first: onMachines, second: onMachines, third: notYetOn + "web on g14"} { + if got := machines(d); !strings.HasPrefix(got, want) { + t.Errorf("%s: %q, want %q", d.ID, got, want) + } + } + reach(sdk.Reach{Repository: "novox/web", Reached: strings.Repeat("9", 40), ReachedSrc: 8}) + if got := machines(first); !strings.Contains(got, "no merge this gate holds") { + t.Errorf("a reached head the gate holds no merge of: %q", got) + } + reach(sdk.Reach{Repository: "novox/web", Holds: []string{"web on ace runs nothing yet"}}) + if got := machines(first); got != notYetOn+"web on ace runs nothing yet" { + t.Errorf("no reached head: %q", got) + } + // The verbs say it; a delivery not merged says nothing of the machines. + reach(sdk.Reach{Repository: "novox/web", Reached: third.MergedAs, ReachedSrc: 9}) + lines := w.h.Deliveries("", "novox/web", "", true) + if len(lines) != 3 || lines[0].Machines != onMachines { + t.Errorf("deliveries = %+v", lines) + } + shown, err := w.h.Show(third.ID) + if err != nil || shown.(map[string]any)["machines"] != onMachines { + t.Errorf("show = %v, %v", shown, err) + } + if got := machines(&Delivery{State: Held, MergedAs: "x", Repository: "novox/web"}); got != "" { + t.Errorf("a held delivery says %q of the machines", got) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go new file mode 100644 index 00000000..85263657 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go @@ -0,0 +1,215 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A person settles a delivery held because its merge reached the trunk without a passing check (novox/hq ADR 0556 clause 5). +// The verb `settle` moves nothing: it writes one ask, for every delivery named, into the gate's asks, which the teller puts to +// the operator as it puts an Approve, its options binding mesh-delivery.settle (sdk.AskVerb). On the answer settle each +// delivery takes the table's asked row held → merged with the operator's words; on leave they stay held. Whether a settled +// merge is on the machines is then derived as for any merge (reached.go). A hold of another kind, or a delivery not held, is +// refused, naming why. + +// The settle ask's two answers, both at the level approve. +const ( + OptionSettle = "settle" + OptionLeave = "leave" +) + +// SettleAsk is the settle ask a delivery is covered by. +type SettleAsk struct { + Key string `json:"key"` + Asking int `json:"asking,omitempty"` + Why string `json:"why"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Answered *time.Time `json:"answered,omitempty"` + Answer string `json:"answer,omitempty"` + Words string `json:"words,omitempty"` +} + +// Settle is the `settle` verb: one ask of the operator for every delivery named, each held for a merge without a passing +// check. Nothing moves until the operator answers. +func (h *Holder) Settle(ids []string, why, by string) (map[string]any, error) { + why = strings.TrimSpace(why) + if why == "" { + return nil, fmt.Errorf("settle says why: it is shown to the operator") + } + if len(ids) == 0 { + return nil, fmt.Errorf("settle names at least one held delivery (ids)") + } + if h.Operator == nil { + return nil, fmt.Errorf("this owner has no way to ask the operator") + } + h.mu.Lock() + var covered []*Delivery + var refused []string + seen := map[string]bool{} + for _, id := range ids { + id = strings.TrimSpace(id) + if seen[id] { + continue + } + seen[id] = true + d := h.deliveries[id] + switch { + case d == nil: + refused = append(refused, id+": no such delivery") + case d.State != Held: + refused = append(refused, fmt.Sprintf("%s: it is %s, not held", id, d.State)) + case !heldUnchecked(d): + refused = append(refused, fmt.Sprintf("%s: it is held for another reason, which settle does not take (%s)", id, d.HeldWhy)) + default: + covered = append(covered, d) + } + } + if len(refused) > 0 { + h.mu.Unlock() + sort.Strings(refused) + return nil, fmt.Errorf("settle refused, nothing asked: %s", strings.Join(refused, "; ")) + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + var names []string + var heads []sdk.Head + for _, d := range covered { + names = append(names, d.ID) + heads = append(heads, sdk.Head{Repository: d.Repository, Number: d.Number, Head: d.Commit}) + } + group := sdk.SettleGroup(names) + key, err := sdk.MergeAskKey(group, heads) + if err != nil { + h.mu.Unlock() + return nil, fmt.Errorf("the settle ask has no key: %w", err) + } + now := h.Now() + // The same deliveries asked again (the ask lapsed, or was left unanswered) are a new asking of the same key, under its own + // router id: the router drops a second ask under an id it kept. + asking := 1 // the first asking (sdk.Ask.Asking: 0 or 1 is the first, n the n-th) + for _, d := range covered { + if d.Settle != nil && d.Settle.Key == key && max(d.Settle.Asking, 1) >= asking { + asking = max(d.Settle.Asking, 1) + 1 + } + } + ask := sdk.Ask{Group: group, Heads: heads, Asking: asking, Ask: composeSettleAsk(sdk.AskingID(key, asking), covered, why, now)} + for _, d := range covered { + d.Settle = &SettleAsk{Key: key, Asking: asking, Why: why, By: by, At: now} + h.keep(d) + } + h.mu.Unlock() + if err := h.Operator.Ask(key, ask); err != nil { + return nil, fmt.Errorf("the settle ask %s could not be written (call settle again): %w", key, err) + } + h.Logf("[mesh-delivery] asked the operator to settle %s: %s", strings.Join(names, ", "), why) + return map[string]any{"asked": key, "deliveries": names, + "said": fmt.Sprintf("The operator is asked once to settle %d deliveries; nothing moves until the answer.", len(covered))}, nil +} + +// settleAct is what a settle ask's answer binds: its verb (mesh-delivery.settle), this ask, the answer and the heads. +func settleAct(group, ask, option string, heads map[string]string) asks.Act { + return asks.Act{"verb": sdk.AskVerb(group), "ask": ask, "option": option, "heads": headsString(heads)} +} + +// composeSettleAsk is the operator's ask to settle these held deliveries. +func composeSettleAsk(id string, ds []*Delivery, why string, now time.Time) asks.Ask { + var a asks.Ask + a.ID, a.Who, a.Expires = id, asks.Operator, now.Add(AskLasts) + a.OnExpiry = "they stay held; settle asks again" + a.About = "delivery.settle" + var pulls, lines []string + heads := map[string]string{} + for _, d := range ds { + pulls = append(pulls, pullSaid(d)) + heads[d.ID] = d.Commit + lines = append(lines, phone(fmt.Sprintf("%s %q merged at %s without a passing check", pullSaid(d), d.Title, + shortOf(d.MergedAs, 12)), 400)) + } + a.Headline = phone(fmt.Sprintf("Settle %d merges made without a passing check", len(ds)), asks.HeadlineLength) + a.Explanation = strings.Join([]string{ + phone(fmt.Sprintf("%s reached main without a passing check and are held. Settle ends the hold: each reads merged, and on the machines once every machine runs it. Leave keeps them held.", + strings.Join(pulls, ", ")), asks.SummaryLength), + phone("Why: "+why, asks.PointLength), + }, "\n") + a.Details = strings.Join(lines, "\n") + group := sdk.SettleGroup(func() []string { + var ids []string + for _, d := range ds { + ids = append(ids, d.ID) + } + return ids + }()) + for _, o := range []struct{ id, label, does string }{ + {OptionSettle, "Settle", "each delivery named ends its hold and reads merged"}, + {OptionLeave, "Leave", "they stay held"}, + } { + binds, _ := asks.ActDigest(settleAct(group, id, o.id, heads)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// isSettleKey is whether an ask key is a settle ask's; under the lock. +func (h *Holder) isSettleKey(key string) bool { + _, group, _, err := sdk.ParseAskKey(key) + return err == nil && strings.HasPrefix(group, sdk.SettleGroupPrefix) +} + +// settleKeys are the settle asks waiting for an answer; under the lock. +func (h *Holder) settleKeys() []string { + seen := map[string]bool{} + var out []string + for _, d := range h.deliveries { + if s := d.Settle; s != nil && s.Answered == nil && d.State == Held && !seen[s.Key] { + seen[s.Key] = true + out = append(out, s.Key) + } + } + sort.Strings(out) + return out +} + +// settled applies the teller's answer to a settle ask; under the lock. The teller held the warrant (the act it bound is +// mesh-delivery.settle); an answer to an earlier asking, or one choosing what the ask did not offer, moves nothing. +func (h *Holder) settled(key string, ans sdk.Answer) { + var covered []*Delivery + for _, d := range h.deliveries { + if d.Settle != nil && d.Settle.Key == key && d.Settle.Answered == nil { + covered = append(covered, d) + } + } + if len(covered) == 0 { + return + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + if asked, at, ok := strings.Cut(ans.Warrant, "@"); ok && at != "" && asked != sdk.AskingID(key, covered[0].Settle.Asking) { + h.Logf("[mesh-delivery] the answer to the settle ask %s was given to another asking (%s): not taken", key, asked) + return + } + if ans.Option != OptionSettle && ans.Option != OptionLeave { + h.refuse(fmt.Errorf("the answer to the settle ask %s chose %s, which it did not offer: nothing settled", key, ans.Option)) + return + } + now := h.Now() + for _, d := range covered { + d.Settle.Answered, d.Settle.Answer, d.Settle.Words = &now, ans.Option, ans.Words + if ans.Option == OptionLeave { + h.Logf("[mesh-delivery] %s: the operator left it held (%s)", d.ID, ans.Words) + h.keep(d) + continue + } + t, err := Apply(d, EvSettled, true, Facts{Now: now, By: "the operator", Why: "settled: " + ans.Words}) + if err != nil { + h.refuse(err) + h.keep(d) + continue + } + d.HeldWhy = "" + h.moved(d, []Transition{t}) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go new file mode 100644 index 00000000..9ccb8cea --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1280 (novox/hq ADR 0556 clause 5): settle writes one ask for several deliveries held for a merge without a passing check and +// moves none of them; it refuses, asking nothing, a delivery held for another reason and one not held, naming why; the +// operator's Leave keeps them held, and Settle, asked again under a new asking, moves each to merged with the operator's words. +func TestSettleAsksTheOperatorOnceAndMovesOnlyOnTheAnswer(t *testing.T) { + w := newWorld(t) + w.op.auto = false + merged := w.now.Add(-time.Hour) + put := func(id string, state State, why string) *Delivery { + repo, commit, _ := strings.Cut(id, "@") + d := &Delivery{ID: id, Repository: repo, Commit: commit, Number: len(id), Title: "a fix", State: state, Since: merged, + MergedAs: "m" + commit, MergedAt: merged, HeldWhy: why} + w.h.mu.Lock() + w.h.keep(d) + w.h.mu.Unlock() + return d + } + a := put("novox/web@aaaaaaaaaaaa", Held, HeldUnchecked) + b := put("novox/sdk@bbbbbbbbbbbb", Held, HeldUnchecked) + put("novox/web@cccccccccccc", Held, "a conflict on the trunk") + put("novox/web@dddddddddddd", Merged, "") + for ids, words := range map[string]string{ + "novox/web@aaaaaaaaaaaa novox/web@cccccccccccc": "held for another reason", + "novox/web@aaaaaaaaaaaa novox/web@dddddddddddd": "not held", + "novox/web@aaaaaaaaaaaa novox/web@eeeeeeeeeeee": "no such delivery", + } { + if _, err := w.h.Settle(strings.Fields(ids), "on every machine since 10-25", aPerson); err == nil || !strings.Contains(err.Error(), words) { + t.Errorf("settle %s: %v, want refused naming %q", ids, err, words) + } + } + if _, err := w.h.Settle([]string{a.ID}, " ", aPerson); err == nil { + t.Error("a settle with no why was asked") + } + if w.op.count() != 0 { + t.Fatalf("a refused settle asked the operator: %d asks", w.op.count()) + } + out, err := w.h.Settle([]string{b.ID, a.ID}, "on every machine since 10-25", aPerson) + if err != nil { + t.Fatal(err) + } + key, _ := out["asked"].(string) + if open := w.op.open(); len(open) != 1 || open[0] != key || w.op.count() != 1 { + t.Fatalf("asks open %v (asked %d), want the one key %s", open, w.op.count(), key) + } + ask := w.op.ask(key) + heads := map[string]string{a.ID: a.Commit, b.ID: b.Commit} + group := sdk.SettleGroup([]string{a.ID, b.ID}) + for _, o := range ask.Options { + if want, _ := asks.ActDigest(settleAct(group, ask.ID, o.ID, heads)); o.Binds != want { + t.Errorf("the option %s binds %s, not mesh-delivery.settle of these heads", o.ID, o.Binds) + } + } + w.h.hearAsks() + if a.State != Held || b.State != Held { + t.Fatalf("moved before an answer: %s, %s", a.State, b.State) + } + w.op.answer(key, OptionLeave) + w.h.hearAsks() + if a.State != Held || a.Settle == nil || a.Settle.Answer != OptionLeave { + t.Fatalf("after Leave: %s %+v", a.State, a.Settle) + } + // Asked again: a new asking of the same key, under its own id. + if _, err := w.h.Settle([]string{a.ID, b.ID}, "asked again", aPerson); err != nil { + t.Fatal(err) + } + if again := w.op.ask(key); again.ID == ask.ID || a.Settle.Asking != 2 { + t.Fatalf("asked again under %s (asking %d), want a new id", again.ID, a.Settle.Asking) + } + w.op.answer(key, OptionSettle) + w.h.hearAsks() + for _, d := range []*Delivery{a, b} { + last := d.Transitions[len(d.Transitions)-1] + if d.State != Merged || last.Event != EvSettled || last.By != "the operator" || !strings.Contains(last.Why, "chose Settle") { + t.Errorf("%s after Settle: %s, %+v", d.ID, d.State, last) + } + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go index f95563a0..bfc56be9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go @@ -77,8 +77,18 @@ const ( EvMerged Event = "merged" // it reached the trunk EvMergedUnchecked Event = "merged-unchecked" // it reached the trunk without a passing check EvTrunkMoved Event = "trunk moved" // another merge moved its trunk, and its verdict names the trunk commit before (novox/hq ADR 0323) + EvSettled Event = "settled" // the operator settled a merge held without a passing check (novox/hq ADR 0556 clause 5) ) +// HeldUnchecked is the why of a delivery held because its merge reached the trunk without a passing check: the one hold a +// person may settle (novox/hq ADR 0556 clause 5). +const HeldUnchecked = "merged without a passing check: only a person decides what follows" + +// heldUnchecked is whether a held delivery is held for a merge without a passing check, and reached the trunk. +func heldUnchecked(d *Delivery) bool { + return d.State == Held && d.MergedAs != "" && strings.HasPrefix(d.HeldWhy, "merged without a passing check") +} + // Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act. type Facts struct { Now time.Time @@ -177,6 +187,20 @@ var Table = []Row{ {From: []State{Proposed, Rejected}, Event: EvMergedUnchecked, To: Held, Guard: "merged without a passing check — a verdict known with it is taken first: only a person decides what follows", Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "", "not merged") }}, + // **A person settles a merge held without a passing check** (novox/hq ADR 0556 clause 5): only on the operator's answer to + // the settle ask, with its words; a hold of another kind is not taken here. Whether it is on the machines is then derived. + {From: []State{Held}, Event: EvSettled, To: Merged, Act: true, + Guard: "held because its merge reached the trunk without a passing check, and the operator answered settle to the ask " + + "`settle` wrote, with the words of that answer", + Holds: func(d *Delivery, f Facts) error { + switch { + case !heldUnchecked(d): + return errors.New("it is not held for a merge without a passing check: " + d.HeldWhy) + case f.By == "" || f.Why == "": + return errors.New("a settle names the operator's answer and its words") + } + return nil + }}, {From: []State{Rejected, Ready}, Event: EvRecheck, To: Proposed, Act: true, Guard: "a person asked, with why, or its group changed", Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "", "a recheck says why") }}, diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go index ab2bccbf..bf92f306 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go @@ -34,6 +34,9 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { case EvTrunkMoved: d.Check = &Verdict{Gate: "pass", Head: "c0ffee000000", Trunk: "d28ca95bbbbb"} d.TrunkNow = "690966dccccccccc" + case EvSettled: + d.MergedAs, d.HeldWhy = "aaaa1111", HeldUnchecked + f.Why, f.By = "settled: the operator chose Settle", "the operator" } return d, f } @@ -103,7 +106,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAccepted, EvRefused, EvRecheck, EvReask, EvLost, EvNewHead, EvClosed, EvMerged, - EvMergedUnchecked, EvTrunkMoved} + EvMergedUnchecked, EvTrunkMoved, EvSettled} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { @@ -128,14 +131,17 @@ func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { // The rules the table is for, read off the table. func TestTheTableKeepsItsRules(t *testing.T) { for _, r := range Table { + // The one row out of a final state is a person's settle of a merge held without a passing check, taken on the + // operator's answer (novox/hq ADR 0556 clause 5): it is already on the trunk, and moves from held to merged. + settle := r.Event == EvSettled && r.Act && len(r.From) == 1 && r.From[0] == Held && r.To == Merged for _, from := range r.From { - // Nothing leaves a final state. - if from.Final() { + // Nothing else leaves a final state. + if from.Final() && !settle { t.Errorf("%s is final and the table leaves it by %s", from, r.Event) } } // Only a ready delivery is merged, and nothing but its merge, or a merge without a passing check, puts one on the trunk. - if r.To == Merged && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) { + if r.To == Merged && !settle && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) { t.Errorf("%v —%s→ merged: only a ready delivery's merge merges", r.From, r.Event) } if r.To == Held && r.Event != EvMergedUnchecked { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 166832cc..07689592 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -22,6 +22,9 @@ type Line struct { Waits string `json:"waits,omitempty"` Owed int `json:"owed,omitempty"` Stalled bool `json:"stalled,omitempty"` + // Machines says, for a merged delivery, whether its merge is on the machines, derived from the delivery view's reach + // (reached.go, novox/hq ADR 0556 clause 4); recorded nowhere. + Machines string `json:"machines,omitempty"` } // waitsFor is what a delivery waits for, in words. @@ -61,7 +64,7 @@ func (h *Holder) Deliveries(state, repository, group string, all bool) []Line { continue } l := Line{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), For: now.Sub(d.Since).Round(time.Second).String(), - Group: d.Group, Number: d.Number, Waits: h.waitsFor(d), Owed: len(d.Owed)} + Group: d.Group, Number: d.Number, Waits: h.waitsFor(d), Owed: len(d.Owed), Machines: h.machines(d)} l.Stalled = pastBound(d, now) out = append(out, l) } @@ -80,6 +83,9 @@ func (h *Holder) Show(id string) (any, error) { defer h.mu.Unlock() if d := h.deliveries[id]; d != nil { out := map[string]any{"delivery": d, "waits": h.waitsFor(d), "check-gates-merge": h.checkGates()} + if m := h.machines(d); m != "" { + out["machines"] = m + } if g := h.groups[d.Group]; g != nil { state, why, _ := h.groupState(g, h.membersOf(g)) out["group"] = map[string]any{"group": g, "state": state, "why": why} @@ -326,5 +332,9 @@ func (h *Holder) Status(listening string) map[string]any { // HeldLine says a held delivery in one line: which, since when and why. A merge without a passing check is a person's to // read; the gate moves nothing after the merge. func HeldLine(d *Delivery) string { - return fmt.Sprintf("%s held since %s: %s", d.ID, d.Since.UTC().Format(time.RFC3339), d.HeldWhy) + line := fmt.Sprintf("%s held since %s: %s", d.ID, d.Since.UTC().Format(time.RFC3339), d.HeldWhy) + if heldUnchecked(d) { + line += " (mesh-delivery.settle asks the operator to settle it, ADR 0556)" + } + return line } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 6dd019be..06e16ec7 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -14,7 +14,8 @@ "table", "stalled", "recheck", - "close" + "close", + "settle" ] } ], @@ -70,7 +71,8 @@ ], "reads": [ "mesh-delivery-teller.answers", - "mesh-delivery-checker.verdicts" + "mesh-delivery-checker.verdicts", + "mesh-delivery-view.reach" ], "tools": [ "delivery_hand_now", From eab69184598657431bb46d7960942d2c6b37c59b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:02:41 +0100 Subject: [PATCH 15/28] The tracker takes the view's per-repository reached head as it takes the whole-mesh one: a merge at or before it is delivered (issue 1280, ADR 0556 clause 6) --- .../cmd/mesh-issues/delivered_test.go | 19 +++++++++++++++++++ .../mesh-issues/cmd/mesh-issues/delivery.go | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/mesh-issues/cmd/mesh-issues/delivered_test.go b/modules/mesh-issues/cmd/mesh-issues/delivered_test.go index cea93929..22a05f06 100644 --- a/modules/mesh-issues/cmd/mesh-issues/delivered_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/delivered_test.go @@ -130,6 +130,25 @@ func TestANewerRevisionCoversTheOlderMergesOfItsRepositoryAsTheForgeSaysAncestry if v, _ := h2.View(n); v.Delivery != "fix merged, not delivered: mesh-catalog#33" { t.Fatalf("another repository's head covered it: %q", v.Delivery) } + // 1280 (ADR 0556 clauses 3 and 6): the view's per-repository form, a repository's reached head with no group, delivers the + // merges at or before that head and not a later one. + h3 := newHarness(t) + reached, later := fixIssue(t, h3, "mesh-catalog#34"), fixIssue(t, h3, "mesh-catalog#35") + mergedFix(t, h3, 34, "feat/e", "6666666666660000") + mergedFix(t, h3, 35, "feat/f", "7777777777770000") + h3.Contains = func(_, _, commit, in string) (bool, error) { + return commit == "6666666666660000" && in == "8888888888880000", nil + } + raw, _ := json.Marshal(map[string]any{"revision": 15, "group": "", "repository": "novox/mesh-catalog", + "heads": map[string]string{"novox/mesh-catalog": "8888888888880000"}}) + if err := h3.OnDelivered(stdio.Envelope{Key: DeliveredEvent, Body: raw}); err != nil { + t.Fatal(err) + } + for n, want := range map[uint64]string{reached: "delivered to every machine it should reach", later: "fix merged, not delivered: mesh-catalog#35"} { + if v, _ := h3.View(n); v.Delivery != want { + t.Errorf("per repository: issue %d reads %q, wanted %q", n, v.Delivery, want) + } + } } // A fix kept with only its merged flag (heard through `checks`, or kept before release 6) is matched to a delivered revision diff --git a/modules/mesh-issues/cmd/mesh-issues/delivery.go b/modules/mesh-issues/cmd/mesh-issues/delivery.go index b09e1fc7..bd21bcfc 100644 --- a/modules/mesh-issues/cmd/mesh-issues/delivery.go +++ b/modules/mesh-issues/cmd/mesh-issues/delivery.go @@ -429,7 +429,8 @@ const DeliveredEvent = "mesh-delivery-view.delivered" // group moved, or whose merge is the head the revision holds for its repository, reached the machines at that revision. The same // revision twice changes nothing (a delivered fix stays delivered). A fix is matched by the group it merged in, by its own merge // commit, or because its merge is an ancestor of the head the revision holds for its repository (coveredBy): a newer revision -// covers the older ones, which the view may never have said. +// covers the older ones, which the view may never have said. The view also says each repository's reached head in this form, +// with no group and that repository's head alone (novox/hq ADR 0556 clause 3): a merge at or before it is on the machines. func (t *Tracker) OnDelivered(e stdio.Envelope) error { var body struct { Revision uint64 `json:"revision"` From 671fdb7dc203612a5a2642ba1c5bd43a9b13d7db Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:05:29 +0100 Subject: [PATCH 16/28] Pin mesh-sdk 33144781 (reach and settle layouts) in the gate (issue 1280) --- modules/mesh-delivery-gate/go.mod | 2 +- modules/mesh-delivery-gate/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 3b7ee541..d219ce5f 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026195749-33144781d652 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index 346bdcce..a8894095 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f h1:9+TZpNq8Ifj1+qmbidE6l/7qkbua4yLdzRMrezNvf1A= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026195749-33144781d652 h1:R+gkczO84KN1nbTG3ubIfAAkbx/jTb+txW1fQVhtmSI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026195749-33144781d652/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 9e41438f1b647fe97920bbd304242150f7197217 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:15:31 +0100 Subject: [PATCH 17/28] A merge of a module running nowhere countable reads not yet on the machines in the gate and the tracker; a settle ask is taken away once answered or lapsed; pin mesh-sdk e44cbaf (review of issue 1280) --- .../cmd/mesh-delivery-gate/approval.go | 1 + .../cmd/mesh-delivery-gate/reached.go | 14 +++++++ .../cmd/mesh-delivery-gate/reached_test.go | 8 ++++ .../cmd/mesh-delivery-gate/settle.go | 37 +++++++++++++++++++ .../cmd/mesh-delivery-gate/settle_test.go | 26 +++++++++++++ modules/mesh-delivery-gate/go.mod | 2 +- modules/mesh-delivery-gate/go.sum | 4 +- .../cmd/mesh-issues/delivered_test.go | 16 ++++++-- .../mesh-issues/cmd/mesh-issues/delivery.go | 16 +++++++- 9 files changed, 116 insertions(+), 8 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index d835ffb4..9e0f7e96 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -299,6 +299,7 @@ func (h *Holder) headsHold(d *Delivery, c *ReadyRun) bool { // §3), and says so once on each pull request it covered (ADR 0421 §6). An ask for a head a newer one replaced ends too. // A Decline stands, and is not ended. Under the lock; the cancels run with the look's asks. func (h *Holder) endStaleAsks() { + h.endStaleSettles() for _, d := range h.deliveries { r := d.Ready if r == nil || r.Merged != nil || d.MergedAs != "" || r.Declined != nil || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go index 5bf5fb37..6456be41 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go @@ -54,6 +54,20 @@ func (h *Holder) machines(d *Delivery) string { switch { case !ok: return notKnownWhere + "the delivery view says no reach of " + d.Repository + } + // A merge that changed a module a node wants and runs nothing countable of is not yet there, whatever the reached head + // (the module holds no head back, ADR 0556 clause 2). + for _, mr := range r.Modules { + if mr.Nodes == 0 || mr.Floor != "" { + continue + } + for _, c := range mr.Changes { + if sameCommit(d.MergedAs, c.Commit) { + return notYetOn + mr.Waiting + } + } + } + switch { case r.Reached == "": return notYetOn + holdsSaid(r) case sameCommit(d.MergedAs, r.Reached): diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go index 89493406..657a42eb 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go @@ -49,6 +49,14 @@ func TestAMergedDeliverySaysWhetherItIsOnTheMachines(t *testing.T) { if got := machines(first); got != notYetOn+"web on ace runs nothing yet" { t.Errorf("no reached head: %q", got) } + // A module a node wants and runs nothing countable of holds no head back, and a merge that changed it is not yet there. + reach(sdk.Reach{Repository: "novox/web", Reached: third.MergedAs, ReachedSrc: 9, Modules: map[string]sdk.ModuleReach{ + "web": {FP: "src2:x", Nodes: 1, Waiting: "web on g14 runs nothing yet", Changes: []sdk.Change{{Commit: second.MergedAs, FP: "src2:x"}}}}}) + for d, want := range map[*Delivery]string{first: onMachines, second: notYetOn + "web on g14 runs nothing yet", third: onMachines} { + if got := machines(d); got != want { + t.Errorf("with a module running nowhere countable, %s: %q, want %q", d.ID, got, want) + } + } // The verbs say it; a delivery not merged says nothing of the machines. reach(sdk.Reach{Repository: "novox/web", Reached: third.MergedAs, ReachedSrc: 9}) lines := w.h.Deliveries("", "novox/web", "", true) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go index 85263657..a95ef111 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go @@ -184,6 +184,7 @@ func (h *Holder) settled(key string, ans sdk.Answer) { } } if len(covered) == 0 { + h.Logf("[mesh-delivery] the answer to the settle ask %s covers no delivery that waits for it: nothing settled", key) return } sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) @@ -212,4 +213,40 @@ func (h *Holder) settled(key string, ans sdk.Answer) { d.HeldWhy = "" h.moved(d, []Transition{t}) } + // Answered: the ask is taken away, as a merge ask is once acted on. + h.cancels = append(h.cancels, h.cancelSettle(key)) +} + +// cancelSettle takes a settle ask away from the gate's asks. +func (h *Holder) cancelSettle(key string) func() { + return func() { + if h.Operator == nil { + return + } + if err := h.Operator.Cancel(key); err != nil { + h.Logf("[mesh-delivery] the settle ask %s could not be taken away (taken away again at the next look): %v", key, err) + } + } +} + +// endStaleSettles takes away every settle ask nobody answered within its bound; its deliveries stay held and settle may ask +// again. Under the lock; the cancels run with the look's asks. +func (h *Holder) endStaleSettles() { + done := map[string]bool{} + for _, d := range h.deliveries { + s := d.Settle + if s == nil || s.Answered != nil || done[s.Key] || h.Now().Before(s.At.Add(AskLasts)) { + continue + } + done[s.Key] = true + h.Logf("[mesh-delivery] the settle ask %s lapsed with no answer: taken away, %s stays held", s.Key, d.ID) + h.cancels = append(h.cancels, h.cancelSettle(s.Key)) + } + for _, d := range h.deliveries { + if s := d.Settle; s != nil && done[s.Key] { + now := h.Now() + s.Answered, s.Answer = &now, "lapsed" + h.keep(d) + } + } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go index 9ccb8cea..bbe88fae 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go @@ -76,12 +76,38 @@ func TestSettleAsksTheOperatorOnceAndMovesOnlyOnTheAnswer(t *testing.T) { if again := w.op.ask(key); again.ID == ask.ID || a.Settle.Asking != 2 { t.Fatalf("asked again under %s (asking %d), want a new id", again.ID, a.Settle.Asking) } + // An answer under a settle key no delivery waits for moves nothing. + other, _ := sdk.MergeAskKey(sdk.SettleGroup([]string{"novox/web@ffffffffffff"}), []sdk.Head{{Repository: "novox/web", Number: 1, Head: "ffffffffffff"}}) + w.h.mu.Lock() + w.h.settled(other, sdk.Answer{Option: OptionSettle, Warrant: "x@now", Words: "chose Settle"}) + w.h.mu.Unlock() + if a.State != Held || b.State != Held || !strings.Contains(w.logbuf.String(), "covers no delivery") { + t.Fatalf("an answer for other ids moved %s, %s", a.State, b.State) + } w.op.answer(key, OptionSettle) w.h.hearAsks() + w.h.mu.Lock() + cancels := len(w.h.cancels) + w.h.mu.Unlock() + if cancels == 0 { + t.Error("the answered settle ask is not taken away") + } for _, d := range []*Delivery{a, b} { last := d.Transitions[len(d.Transitions)-1] if d.State != Merged || last.Event != EvSettled || last.By != "the operator" || !strings.Contains(last.Why, "chose Settle") { t.Errorf("%s after Settle: %s, %+v", d.ID, d.State, last) } } + // A settle ask nobody answers within its bound is taken away; its delivery stays held. + c := put("novox/web@999999999999", Held, HeldUnchecked) + if _, err := w.h.Settle([]string{c.ID}, "once more", aPerson); err != nil { + t.Fatal(err) + } + w.later(AskLasts + time.Minute) + w.h.mu.Lock() + w.h.endStaleSettles() + w.h.mu.Unlock() + if c.State != Held || c.Settle.Answer != "lapsed" { + t.Errorf("a lapsed settle ask: %s %+v", c.State, c.Settle) + } } diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index d219ce5f..1a02ef85 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026195749-33144781d652 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026201248-e44cbaf57fc3 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index a8894095..73a8b6b4 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026195749-33144781d652 h1:R+gkczO84KN1nbTG3ubIfAAkbx/jTb+txW1fQVhtmSI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026195749-33144781d652/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026201248-e44cbaf57fc3 h1:kOAR8GaTpOP9j2XNm3grqEfSwSDHnWbHKd+Qftkr/QE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026201248-e44cbaf57fc3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-issues/cmd/mesh-issues/delivered_test.go b/modules/mesh-issues/cmd/mesh-issues/delivered_test.go index 22a05f06..67a810be 100644 --- a/modules/mesh-issues/cmd/mesh-issues/delivered_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/delivered_test.go @@ -139,11 +139,19 @@ func TestANewerRevisionCoversTheOlderMergesOfItsRepositoryAsTheForgeSaysAncestry h3.Contains = func(_, _, commit, in string) (bool, error) { return commit == "6666666666660000" && in == "8888888888880000", nil } - raw, _ := json.Marshal(map[string]any{"revision": 15, "group": "", "repository": "novox/mesh-catalog", - "heads": map[string]string{"novox/mesh-catalog": "8888888888880000"}}) - if err := h3.OnDelivered(stdio.Envelope{Key: DeliveredEvent, Body: raw}); err != nil { - t.Fatal(err) + say := func(rev int, notYet ...string) { + raw, _ := json.Marshal(map[string]any{"revision": rev, "group": "", "repository": "novox/mesh-catalog", + "heads": map[string]string{"novox/mesh-catalog": "8888888888880000"}, "not_yet": notYet}) + if err := h3.OnDelivered(stdio.Envelope{Key: DeliveredEvent, Body: raw}); err != nil { + t.Fatal(err) + } } + // A merge the event says is not yet on the machines (its module runs on no node yet) waits for the next event. + say(15, "6666666666660000") + if v, _ := h3.View(reached); v.Delivery != "fix merged, not delivered: mesh-catalog#34" { + t.Errorf("a merge said not yet: %q", v.Delivery) + } + say(15) for n, want := range map[uint64]string{reached: "delivered to every machine it should reach", later: "fix merged, not delivered: mesh-catalog#35"} { if v, _ := h3.View(n); v.Delivery != want { t.Errorf("per repository: issue %d reads %q, wanted %q", n, v.Delivery, want) diff --git a/modules/mesh-issues/cmd/mesh-issues/delivery.go b/modules/mesh-issues/cmd/mesh-issues/delivery.go index bd21bcfc..75812f9f 100644 --- a/modules/mesh-issues/cmd/mesh-issues/delivery.go +++ b/modules/mesh-issues/cmd/mesh-issues/delivery.go @@ -438,6 +438,9 @@ func (t *Tracker) OnDelivered(e stdio.Envelope) error { Heads map[string]string `json:"heads"` // At is when the view judged the revision delivered, when it says so; else this tracker's moment of hearing it. At time.Time `json:"at"` + // NotYet are, in the per-repository form, merges at or before the head that are not yet on the machines (a module a node + // wants and runs nothing countable of, ADR 0556 clause 2): a fix merged as one of them waits for a later event. + NotYet []string `json:"not_yet"` } if err := json.Unmarshal(e.Body, &body); err != nil || (body.Group == "" && len(body.Heads) == 0) { return nil @@ -457,7 +460,8 @@ func (t *Tracker) OnDelivered(e stdio.Envelope) error { if f != nil && f.Merged && !contains(is.Delivered, pr) && f.MergedAs == "" { t.fillMerge(n, f, pr) } - if f == nil || !f.Merged || contains(is.Delivered, pr) || !(deliveredBy(f, body.Group, body.Heads) || t.coveredBy(f, body.Heads)) { + if f == nil || !f.Merged || contains(is.Delivered, pr) || notYet(f, body.NotYet) || + !(deliveredBy(f, body.Group, body.Heads) || t.coveredBy(f, body.Heads)) { continue } pr := pr @@ -585,6 +589,16 @@ func deliveredBy(f *Fix, group string, heads map[string]string) bool { return false } +// notYet is whether a fix's merge is among the merges an event says are not yet on the machines. +func notYet(f *Fix, merges []string) bool { + for _, m := range merges { + if f.MergedAs != "" && sameCommitIn(f.MergedAs, m) { + return true + } + } + return false +} + // sameCommitIn is whether two commits are one, however either is abbreviated (at least seven characters). func sameCommitIn(a, b string) bool { a, b = strings.ToLower(a), strings.ToLower(b) From 3f8b0b42480b6eef362f2360ec5d9d0c2e088b7d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:19:59 +0100 Subject: [PATCH 18/28] Build modules take asks only within their node's build hours mesh-build-go, -node and -image declare build-hours (node=HH:MM-HH:MM in the node's local time, empty = any hour) and hand it to the SDK puller (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00); their SDK pin moves to mesh-sdk d00dac1, which has it. --- modules/mesh-build-go/cmd/mesh-build-go/asks.go | 6 +++++- modules/mesh-build-go/cmd/mesh-build-go/asks_test.go | 7 +++++++ modules/mesh-build-go/files/settings.json | 3 ++- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 4 ++-- modules/mesh-build-go/module.json | 8 +++++++- modules/mesh-build-image/cmd/mesh-build-image/asks.go | 6 +++++- modules/mesh-build-image/files/settings.json | 3 ++- modules/mesh-build-image/go.mod | 2 +- modules/mesh-build-image/go.sum | 4 ++-- modules/mesh-build-image/module.json | 8 +++++++- modules/mesh-build-node/cmd/mesh-build-node/asks.go | 6 +++++- modules/mesh-build-node/cmd/mesh-build-node/asks_test.go | 7 +++++++ modules/mesh-build-node/files/settings.json | 3 ++- modules/mesh-build-node/go.mod | 2 +- modules/mesh-build-node/go.sum | 4 ++-- modules/mesh-build-node/module.json | 8 +++++++- 17 files changed, 65 insertions(+), 18 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 722d5b48..816793b2 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -86,8 +86,12 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep var watch func(string, func(stdio.ProvisionChange) error) error // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + // And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes + // nothing new. var cores func() float64 + var hours func() (*buildstep.BuildWindow, string) if capacity == nil { + hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say) capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskGo], m.Say) @@ -101,7 +105,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep } } return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, Logf: m.Say, Has: m.Has, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Has: m.Has, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 02f18241..20f6d788 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -137,6 +137,13 @@ func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskGo] { t.Errorf("the settings template holds cores-per-step %g", got) } + // And the build hours (the operator, 2026-10-26): a setting of their own, empty by default (any hour), in the template. + if hours := string(doc.Config[buildstep.BuildHoursSetting].Default); hours != `""` { + t.Errorf("the build-hours default = %s, want empty", hours) + } + if w, fault := buildstep.NodeBuildHours(moduleDir+"/files/settings.json", "g14", t.Errorf)(); w != nil || fault != "" { + t.Errorf("the settings template holds build hours %v (%s)", w, fault) + } var capacity string _ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity) if capacity != buildstep.DefaultCapacities { diff --git a/modules/mesh-build-go/files/settings.json b/modules/mesh-build-go/files/settings.json index 054af9a2..56bb28d4 100644 --- a/modules/mesh-build-go/files/settings.json +++ b/modules/mesh-build-go/files/settings.json @@ -1,4 +1,5 @@ { "capacity": "shanks=3,ace=2,g14=0,novox=0", - "cores-per-step": 6 + "cores-per-step": 6, + "build-hours": "" } diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 5e2a7a39..24e99464 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index b4a5f723..4fd3d8a8 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 h1:mSBxF9KgIJyNL0gOKYq5naWewUj2yazU/tGOh12Tze0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03 h1:QIcGyV96AHZ6t/4AgBDlUOcs5tq40PppC7pwEpmzZco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 9530f128..7526d1a0 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -30,6 +30,12 @@ "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" }, + "build-hours": { + "type": "string", + "description": "When each node may take this module's build asks, as node=HH:MM-HH:MM pairs in the node's local time, separated by commas, such as g14=18:00-06:00 (an end before its start crosses midnight). Outside its window a node takes no new step and finishes the ones it runs; a node not named, or an empty value, builds at any hour. The operator, 2026-10-26: g14 builds only outside office hours.", + "pattern": "|[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9](,[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9])*", + "default": "" + }, "cores-per-step": { "type": "number", "description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", @@ -113,7 +119,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-go", - "takes-config": ["capacity", "cores-per-step"], + "takes-config": ["capacity", "cores-per-step", "build-hours"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index 3775fc7b..ad9899bf 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -42,8 +42,12 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep var watch func(string, func(stdio.ProvisionChange) error) error // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + // And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes + // nothing new. var cores func() float64 + var hours func() (*buildstep.BuildWindow, string) if capacity == nil { + hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say) capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskImage], m.Say) @@ -54,7 +58,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-image/files/settings.json b/modules/mesh-build-image/files/settings.json index 9acf4e20..73ee4fa8 100644 --- a/modules/mesh-build-image/files/settings.json +++ b/modules/mesh-build-image/files/settings.json @@ -1,4 +1,5 @@ { "capacity": "shanks=3,ace=2,g14=0,novox=0", - "cores-per-step": 2 + "cores-per-step": 2, + "build-hours": "" } diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 0e530690..e931caa0 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index 9e114f39..cffc5072 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 h1:mSBxF9KgIJyNL0gOKYq5naWewUj2yazU/tGOh12Tze0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03 h1:QIcGyV96AHZ6t/4AgBDlUOcs5tq40PppC7pwEpmzZco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 3fe1377f..c83b9d88 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -25,6 +25,12 @@ "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" }, + "build-hours": { + "type": "string", + "description": "When each node may take this module's build asks, as node=HH:MM-HH:MM pairs in the node's local time, separated by commas, such as g14=18:00-06:00 (an end before its start crosses midnight). Outside its window a node takes no new step and finishes the ones it runs; a node not named, or an empty value, builds at any hour. The operator, 2026-10-26: g14 builds only outside office hours.", + "pattern": "|[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9](,[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9])*", + "default": "" + }, "cores-per-step": { "type": "number", "description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", @@ -84,7 +90,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-image", - "takes-config": ["capacity", "cores-per-step"], + "takes-config": ["capacity", "cores-per-step", "build-hours"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index 25bff14f..f2353133 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -50,8 +50,12 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep var watch func(string, func(stdio.ProvisionChange) error) error // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + // And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes + // nothing new. var cores func() float64 + var hours func() (*buildstep.BuildWindow, string) if capacity == nil { + hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say) capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskNode], m.Say) @@ -62,7 +66,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskNode, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Prepare(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go index dcecf188..8ae9a43d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go @@ -95,6 +95,13 @@ func TestManifestTakesTheBuildCapacity(t *testing.T) { if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskNode] { t.Errorf("the settings template holds cores-per-step %g", got) } + // And the build hours (the operator, 2026-10-26): a setting of their own, empty by default (any hour), in the template. + if hours := string(doc.Config[buildstep.BuildHoursSetting].Default); hours != `""` { + t.Errorf("the build-hours default = %s, want empty", hours) + } + if w, fault := buildstep.NodeBuildHours(moduleDir+"/files/settings.json", "g14", t.Errorf)(); w != nil || fault != "" { + t.Errorf("the settings template holds build hours %v (%s)", w, fault) + } var capacity string _ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity) if capacity != buildstep.DefaultCapacities { diff --git a/modules/mesh-build-node/files/settings.json b/modules/mesh-build-node/files/settings.json index 9acf4e20..73ee4fa8 100644 --- a/modules/mesh-build-node/files/settings.json +++ b/modules/mesh-build-node/files/settings.json @@ -1,4 +1,5 @@ { "capacity": "shanks=3,ace=2,g14=0,novox=0", - "cores-per-step": 2 + "cores-per-step": 2, + "build-hours": "" } diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index 4f5fd29a..55492ff6 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 8b19ae3a..338460ec 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642 h1:mSBxF9KgIJyNL0gOKYq5naWewUj2yazU/tGOh12Tze0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026185254-02f50bd9d642/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03 h1:QIcGyV96AHZ6t/4AgBDlUOcs5tq40PppC7pwEpmzZco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202137-d00dac140b03/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 970cd694..baf8b291 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -26,6 +26,12 @@ "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" }, + "build-hours": { + "type": "string", + "description": "When each node may take this module's build asks, as node=HH:MM-HH:MM pairs in the node's local time, separated by commas, such as g14=18:00-06:00 (an end before its start crosses midnight). Outside its window a node takes no new step and finishes the ones it runs; a node not named, or an empty value, builds at any hour. The operator, 2026-10-26: g14 builds only outside office hours.", + "pattern": "|[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9](,[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9])*", + "default": "" + }, "cores-per-step": { "type": "number", "description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", @@ -106,7 +112,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-node", - "takes-config": ["capacity", "cores-per-step"], + "takes-config": ["capacity", "cores-per-step", "build-hours"], "merge": "json", "template": "files/settings.json" }, From dcf32576a88071eadf7f7f13f37f62d7bfc0dc01 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:25:24 +0100 Subject: [PATCH 19/28] mesh-build-image: build with the module's directory as context, so a Dockerfile in a subdirectory finds its COPY paths (issue 1703) photos' admin-client, client and server Dockerfiles live in subdirectories and copy paths from the module's root (admin-client/..., client-server/serve.mjs), as build-agent built them and as the fingerprint reads the module's whole directory. Taking the Dockerfile's own directory as context failed every such COPY with copier: stat "/admin-client/package.json": no such file or directory, on shanks and on ace. Quick fix (rejected): move photos' Dockerfiles to its repository root; it changes the operator's photo application and leaves the trap for the next module. Thorough fix (chosen): one context for every image, the module's directory; it never leaves that directory, so the isolation of ADR 0487 section 2 holds. --- modules/mesh-build-image/README.md | 5 ++- .../cmd/mesh-build-image/build.go | 18 +++++--- .../cmd/mesh-build-image/build_test.go | 41 ++++++++++++++++++- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/modules/mesh-build-image/README.md b/modules/mesh-build-image/README.md index a6c3b62e..e27eaa30 100644 --- a/modules/mesh-build-image/README.md +++ b/modules/mesh-build-image/README.md @@ -24,8 +24,9 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re - `build_image_fetch` fetches a commit over SSH into a workspace (a base commit makes the merge preview, as the other build modules' fetch). Long: answers a step id and ends on its key in `steps`. -- `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile of the module's directory as file - and context, `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With +- `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile `from` names (in the module's + directory or a subdirectory of it) as file and the module's own directory as context, so a COPY names paths from the + module's root (issue 1703), `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With `publish: true` (a merged trunk commit only) the module's own process then pushes it to the registry under `/:` (the first 12 hex digits of the digest unless the call gives one), waiting out a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build.go b/modules/mesh-build-image/cmd/mesh-build-image/build.go index c3e1d0e4..323cf3a5 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build.go @@ -55,18 +55,18 @@ func (m *Module) buildTool() StepTool { } // unsupportedImage says why this module does not build the artifact, or "" when it does. It is the one place that -// knows which image forms are out: ADR 0487 §1 takes the Dockerfile's directory as the build context and the base +// knows which image forms are out: ADR 0487 §1 builds from the module's own directory (the build context) and the base // from the Dockerfile or an image pinned in `build.on`, so a context that is another repository and a base that is // another module's artifact are not built (the lead's decision for R4c-9, which retires them with build-agent at R4g). func unsupportedImage(manifest catalogue.Manifest, a catalogue.Artifact) string { if a.Context != nil { - return fmt.Sprintf("the image %s of %s builds with the repository %s as its context; mesh-build-image takes the Dockerfile's directory as the build context (ADR 0487 §1) and does not build a whole repository", + return fmt.Sprintf("the image %s of %s builds with the repository %s as its context; mesh-build-image takes the module's own directory as the build context (ADR 0487 §1) and does not build another repository", a.Name, manifest.Module, a.Context.Repository) } if manifest.Build != nil { for _, on := range manifest.Build.On { if on.Module != "" { - return fmt.Sprintf("the image %s of %s stands on the artifact %s of the module %s through the build argument %s; mesh-build-image builds only from the Dockerfile's directory and an image pinned in build.on (ADR 0487 §1)", + return fmt.Sprintf("the image %s of %s stands on the artifact %s of the module %s through the build argument %s; mesh-build-image builds only from the module's own directory and an image pinned in build.on (ADR 0487 §1)", a.Name, manifest.Module, on.Artifact, on.Module, on.Arg) } } @@ -183,8 +183,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return m.push(ctx, workspace, manifest.Module, local, res, tag) } -// dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the directory it lies in. It must -// be a regular file inside the module's own directory (checks/imageplace holds the catalogue to the same). +// dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the module's own directory, the +// one its manifest lies in, whichever subdirectory the Dockerfile sits in. `from` is a path relative to that +// directory, and so is every COPY of the Dockerfile: a recipe in a subdirectory (photos' admin-client/Dockerfile +// copies admin-client/... and client-server/serve.mjs) builds as build-agent built it, and as the fingerprint reads +// the module's whole directory as its build source. Taking the Dockerfile's own directory instead failed every such +// COPY with `copier: stat ... no such file or directory` (issue 1703, 2026-10-26). The context never leaves the module's +// directory, so a module's build reads no other module's files. The Dockerfile must be a regular file inside that +// directory (checks/imageplace holds the catalogue to the same). func dockerfileOf(moduleDir, from string) (dockerfile, contextDir string, err error) { clean := filepath.Clean(filepath.FromSlash(strings.TrimSpace(from))) if from == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { @@ -195,7 +201,7 @@ func dockerfileOf(moduleDir, from string) (dockerfile, contextDir string, err er if err != nil || !info.Mode().IsRegular() { return "", "", fmt.Errorf("%q is not a file in the module's directory", from) } - return dockerfile, filepath.Dir(dockerfile), nil + return dockerfile, moduleDir, nil } // localName names the image in buildah's store: one name for each artifact of a workspace. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build_test.go b/modules/mesh-build-image/cmd/mesh-build-image/build_test.go index 3bb92b3a..a8c866ac 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build_test.go @@ -94,7 +94,7 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) { } } if !strings.HasSuffix(bud[0].line, " "+dir) { - t.Errorf("the context is not the Dockerfile's directory:\n%s", bud[0].line) + t.Errorf("the context is not the module's directory:\n%s", bud[0].line) } for _, c := range f.calls { if c.name != "buildah" { @@ -122,6 +122,43 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) { } } +// Issue 1703, 2026-10-26: a Dockerfile in a subdirectory of the module builds with the module's directory as its +// context, so its COPY of a path from the module's root (photos' admin-client/Dockerfile copies admin-client/... and +// client-server/serve.mjs) finds it; the Dockerfile's own directory as the context failed with `copier: stat +// "/admin-client/package.json": no such file or directory`. +func TestBuildContextIsTheModulesDirectoryForADockerfileInASubdirectory(t *testing.T) { + m, f, states := setup(t, func(s string) string { + return strings.Replace(s, `"from": "Dockerfile"`, `"from": "admin-client/Dockerfile"`, 1) + }) + dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") + for name, body := range map[string]string{ + "admin-client/Dockerfile": "ARG DEMO_BASE\nFROM ${DEMO_BASE} AS final\nCOPY admin-client/package.json client-server/serve.mjs /\n", + "admin-client/package.json": "{}\n", + "client-server/serve.mjs": "\n", + } { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + _, st := runBuild(t, m, states, nil) + if st.State != delivery.StepPassed { + t.Fatalf("step = %+v", st) + } + bud := f.called("bud") + if len(bud) != 1 { + t.Fatalf("buildah bud ran %d times", len(bud)) + } + if !strings.Contains(bud[0].line, "--file "+dir+"/admin-client/Dockerfile ") { + t.Errorf("the Dockerfile is not the subdirectory's:\n%s", bud[0].line) + } + if !strings.HasSuffix(bud[0].line, " "+dir) { + t.Errorf("the context is not the module's directory, so admin-client/ and client-server/ are out of reach:\n%s", bud[0].line) + } +} + // 1703c, issue 1703: a failing Dockerfile step fails the step, its reason saying buildah's error from stderr, never the // last STEP line buildah writes on stdout; nothing is pushed. func TestBuildFailsWithBuildahsError(t *testing.T) { @@ -178,7 +215,7 @@ func TestBuildRefusesAnImageWithARepositoryContextOrAModuleBase(t *testing.T) { }{ "context": {func(s string) string { return strings.Replace(s, `"target": "final"`, `"target": "final", "context": {"seat": "git", "repository": "novox/mesh-controller"}`, 1) - }, []string{"novox/mesh-controller", "Dockerfile's directory", "ADR 0487"}}, + }, []string{"novox/mesh-controller", "module's own directory", "ADR 0487"}}, "module base": {func(s string) string { return strings.Replace(s, `"image": "docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001"`, `"module": "mesh-sdk", "artifact": "lib"`, 1) }, []string{"artifact lib of the module mesh-sdk", "DEMO_BASE"}}, From be7e98d25cd93172c9ee9fad662471a3c29f391e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:59:21 +0100 Subject: [PATCH 20/28] Pin mesh-sdk 5f45175 (l2-next: feat/1126-nodes-compose merged with sdk main) after merging main (issue 1126) --- checks/placement/go.mod | 2 +- checks/placement/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/checks/placement/go.mod b/checks/placement/go.mod index 29fdcc2b..902a02d4 100644 --- a/checks/placement/go.mod +++ b/checks/placement/go.mod @@ -2,4 +2,4 @@ module checks/placement go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026205647-5f45175ed02b diff --git a/checks/placement/go.sum b/checks/placement/go.sum index 5c5cbdbe..18b1e59c 100644 --- a/checks/placement/go.sum +++ b/checks/placement/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 h1:RInAboHBk5THYf6Agvy05iO9p2XV5zGb0ilYa+iSO7o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026205647-5f45175ed02b h1:6iRVEZi6HUi9r2pFPz8aMLJy6FPdnyTOZ79/a7eSbNI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026205647-5f45175ed02b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= From ba49a7f0f9360361894372087f0a6364f8247a70 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:02:58 +0100 Subject: [PATCH 21/28] mesh-build-go: a reads-outside marker lists what its tests read, so its package is kept until one of those files changes (issue 1936) 47 packages carried the marker and were never kept: about 2000 tests per level ran on every check, whatever it changed. The marker now lists, as paths or globs from the repository's root (? before a path whose absence is read), what the package's tests read outside it; the digest hashes those files with the package's own non-Go files. A marker that lists nothing, names an unclean path, or a line that matches nothing keeps today's never-kept digest. The 46 markers list their reads; checks/deps and checks/trunk lose theirs (their tests read only temporary directories). Digest form 6. Quick fix of issue 1936, chosen by the operator 2026-10-26 21:56; the thorough fix (the Go test log's files kept with each verdict) stays on the issue. --- checks/agenthome/reads-outside | 6 +- checks/config/reads-outside | 6 +- checks/deps/reads-outside | 1 - checks/health/reads-outside | 6 +- checks/imagebuild/reads-outside | 6 +- checks/imageplace/reads-outside | 6 +- checks/locks/reads-outside | 8 +- checks/mailstore/reads-outside | 6 +- checks/nobinaries/reads-outside | 6 +- checks/noscripts/reads-outside | 8 +- checks/owned/reads-outside | 6 +- checks/particulars/reads-outside | 7 +- checks/paths/reads-outside | 6 +- checks/predefined/reads-outside | 7 +- checks/secrets/reads-outside | 6 +- checks/sshdir/reads-outside | 6 +- checks/templates/reads-outside | 6 +- checks/toolchains/reads-outside | 6 +- checks/trunk/reads-outside | 1 - checks/words/reads-outside | 8 +- .../adwaita/internal/desktop/reads-outside | 10 +- .../blueman/cmd/blueman-tools/reads-outside | 26 ++- modules/brother/cmd/brother/reads-outside | 7 +- .../cmd/forticlient-tools/reads-outside | 26 ++- modules/i3/internal/desktop/reads-outside | 10 +- modules/lemurs/internal/desktop/reads-outside | 10 +- .../cmd/mesh-build-go/beside_digest_test.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/digest.go | 154 ++++++++++++++++-- .../cmd/mesh-build-go/digest_test.go | 75 +++++++-- .../cmd/mesh-build-go/reads-outside | 8 +- .../cmd/mesh-build-image/reads-outside | 11 +- .../cmd/mesh-build-node/reads-outside | 6 +- .../cmd/mesh-delivery-gate/reads-outside | 6 +- .../mesh-issues/cmd/mesh-issues/reads-outside | 8 +- .../cmd/mesh-knowledge/reads-outside | 6 +- .../cmd/mesh-mail-notifier/reads-outside | 6 +- .../cmd/mesh-mail-reader/reads-outside | 6 +- modules/mesh-mail/cmd/mesh-mail/reads-outside | 6 +- .../cmd/mesh-provisioner-client/reads-outside | 8 +- modules/minio/cmd/object-store/reads-outside | 6 +- .../cmd/uplink-tools/reads-outside | 7 +- .../cmd/nextcloud-client-tools/reads-outside | 26 ++- .../cmd/nm-applet-tools/reads-outside | 26 ++- .../cmd/openrazer-tools/reads-outside | 26 ++- modules/pacman/cmd/pacman-tools/reads-outside | 6 +- .../cmd/polychromatic-tools/reads-outside | 26 ++- .../cmd/postgres-provider/reads-outside | 6 +- modules/slack/cmd/slack-tools/reads-outside | 26 ++- .../cmd/uplink-tools/reads-outside | 7 +- modules/xorg/internal/desktop/reads-outside | 10 +- modules/xterm/internal/desktop/reads-outside | 10 +- 51 files changed, 613 insertions(+), 78 deletions(-) delete mode 100644 checks/deps/reads-outside delete mode 100644 checks/trunk/reads-outside diff --git a/checks/agenthome/reads-outside b/checks/agenthome/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/agenthome/reads-outside +++ b/checks/agenthome/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/config/reads-outside b/checks/config/reads-outside index f6b2d871..cc3c0a39 100644 --- a/checks/config/reads-outside +++ b/checks/config/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and the files it names. +modules diff --git a/checks/deps/reads-outside b/checks/deps/reads-outside deleted file mode 100644 index f6b2d871..00000000 --- a/checks/deps/reads-outside +++ /dev/null @@ -1 +0,0 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/health/reads-outside b/checks/health/reads-outside index f6b2d871..1aaa7f89 100644 --- a/checks/health/reads-outside +++ b/checks/health/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest. +modules/*/module.json diff --git a/checks/imagebuild/reads-outside b/checks/imagebuild/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/imagebuild/reads-outside +++ b/checks/imagebuild/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/imageplace/reads-outside b/checks/imageplace/reads-outside index f6b2d871..8a080f79 100644 --- a/checks/imageplace/reads-outside +++ b/checks/imageplace/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest, and whether the path each image artifact builds from is there. +modules diff --git a/checks/locks/reads-outside b/checks/locks/reads-outside index f6b2d871..4f2c2396 100644 --- a/checks/locks/reads-outside +++ b/checks/locks/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest, a TypeScript module's package.json and whether its lock is there. +modules/*/module.json +modules/*/package.json +modules/*/package-lock.json diff --git a/checks/mailstore/reads-outside b/checks/mailstore/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/mailstore/reads-outside +++ b/checks/mailstore/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/nobinaries/reads-outside b/checks/nobinaries/reads-outside index f6b2d871..8c970936 100644 --- a/checks/nobinaries/reads-outside +++ b/checks/nobinaries/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every file under modules/ (git ls-files and a walk). +modules diff --git a/checks/noscripts/reads-outside b/checks/noscripts/reads-outside index f6b2d871..a161a850 100644 --- a/checks/noscripts/reads-outside +++ b/checks/noscripts/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# that none of the old check scripts stands at the root. +?merge-check.sh +?merge-check-typescript.sh +?check-nested-modules.sh diff --git a/checks/owned/reads-outside b/checks/owned/reads-outside index f6b2d871..bc374f72 100644 --- a/checks/owned/reads-outside +++ b/checks/owned/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest; its lists are this package's own files. +modules/*/module.json diff --git a/checks/particulars/reads-outside b/checks/particulars/reads-outside index f6b2d871..c2a9b667 100644 --- a/checks/particulars/reads-outside +++ b/checks/particulars/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# check("../.."): the root's list and every file of the modules it names. +particulars +modules diff --git a/checks/paths/reads-outside b/checks/paths/reads-outside index f6b2d871..cfaaf0f8 100644 --- a/checks/paths/reads-outside +++ b/checks/paths/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and every file of a module. +modules diff --git a/checks/predefined/reads-outside b/checks/predefined/reads-outside index f6b2d871..81a14658 100644 --- a/checks/predefined/reads-outside +++ b/checks/predefined/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the root's list of predefined keys and every manifest that names one. +predefined.json +modules/*/module.json diff --git a/checks/secrets/reads-outside b/checks/secrets/reads-outside index f6b2d871..cc3c0a39 100644 --- a/checks/secrets/reads-outside +++ b/checks/secrets/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and the files it names. +modules diff --git a/checks/sshdir/reads-outside b/checks/sshdir/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/sshdir/reads-outside +++ b/checks/sshdir/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/templates/reads-outside b/checks/templates/reads-outside index f6b2d871..e854eebe 100644 --- a/checks/templates/reads-outside +++ b/checks/templates/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and its templates. +modules diff --git a/checks/toolchains/reads-outside b/checks/toolchains/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/toolchains/reads-outside +++ b/checks/toolchains/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/trunk/reads-outside b/checks/trunk/reads-outside deleted file mode 100644 index f6b2d871..00000000 --- a/checks/trunk/reads-outside +++ /dev/null @@ -1 +0,0 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/words/reads-outside b/checks/words/reads-outside index f6b2d871..035d1d78 100644 --- a/checks/words/reads-outside +++ b/checks/words/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# run("../.."): the root's lists of retired words and lines, and every module's texts. +retired-words +retired-lines +modules diff --git a/modules/adwaita/internal/desktop/reads-outside b/modules/adwaita/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/adwaita/internal/desktop/reads-outside +++ b/modules/adwaita/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/blueman/cmd/blueman-tools/reads-outside b/modules/blueman/cmd/blueman-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/blueman/cmd/blueman-tools/reads-outside +++ b/modules/blueman/cmd/blueman-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/brother/cmd/brother/reads-outside b/modules/brother/cmd/brother/reads-outside index f6b2d871..95611a5c 100644 --- a/modules/brother/cmd/brother/reads-outside +++ b/modules/brother/cmd/brother/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# mesh-mail's store.json is the same as brother's; no module scan is left. +modules/mesh-mail/files/store.json +?modules/scan diff --git a/modules/forticlient/cmd/forticlient-tools/reads-outside b/modules/forticlient/cmd/forticlient-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/forticlient/cmd/forticlient-tools/reads-outside +++ b/modules/forticlient/cmd/forticlient-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/i3/internal/desktop/reads-outside b/modules/i3/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/i3/internal/desktop/reads-outside +++ b/modules/i3/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/lemurs/internal/desktop/reads-outside b/modules/lemurs/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/lemurs/internal/desktop/reads-outside +++ b/modules/lemurs/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index 22aebaca..f59242f7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) { t.Errorf("the integration digest listed %q", s.command) } } - if DigestVersion != "5" { + if DigestVersion != "6" { t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 746a6f85..ae275800 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -16,6 +16,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "sort" "strconv" @@ -29,17 +30,24 @@ import ( // commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement). // 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1). // 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935). -const DigestVersion = "5" +// 6: a reads-outside marker that lists what its tests read enters the digest with those files' contents (issue 1936). +const DigestVersion = "6" // besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file // depends on the siblings' commits. const besideVariable = "MESH_CHECK_BESIDE" // ReadsOutsideMarker is the name of a file in a package's directory that says its tests read files outside it (a -// repository's documents, a sibling directory): such a package is never kept (issue 1935). The digest hashes only what -// the Go toolchain lists, so it cannot name what a test opens elsewhere; a marker file, beside the Go files where a -// reader of the package sees it, is the plain way to say so until the checker keeps the files a test opened (the Go -// test log) with each verdict. +// repository's documents, a sibling directory), which the Go toolchain does not list (issue 1935). It lists, one per +// line, what those tests read, as paths from the repository's root; a path may be a glob (filepath.Match), and a +// directory it names is read whole (issue 1936). Blank lines and lines starting with # are not read. A line starting +// with ? names a path whose absence is what the tests read (a file that must not be there): it may match nothing. +// +// The digest then hashes, besides what the toolchain lists, every line of the marker, every file it names (under its +// path in the tree, by content) and every file of the package's own directory. A marker that lists nothing, a line +// that is not a clean relative path, or a line without ? that matches nothing makes the package one that is never kept +// (its digest a fresh random hex): a list nobody can trust keeps nothing. The exact fix, the files a test opened kept +// from the Go test log with each verdict, replaces the lists (issue 1936). const ReadsOutsideMarker = "reads-outside" // NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need @@ -128,9 +136,13 @@ type DigestResult struct { // only when they are the siblings it gave the fetch. Beside []string `json:"beside,omitempty"` WithoutTests []string `json:"without_tests,omitempty"` - // ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh - // random one), so the check runs it every time. + // ReadsOutside names the packages whose marker lists nothing it can trust (ReadsOutsideMarker): each has a digest + // that no kept verdict can have (a fresh random one), so the check runs it every time. ReadsOutside []string `json:"reads_outside,omitempty"` + // neverKept is, per package of ReadsOutside, why its marker cannot be trusted; hashedFiles caches the content lines + // of the files markers name, which several packages share (the catalogue's manifests). + neverKept map[string]string + hashedFiles map[string]string // Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it. Needs []delivery.PackageNeeds `json:"needs,omitempty"` // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the @@ -247,7 +259,7 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges sort.Slice(res.Needs, func(i, j int) bool { return res.Needs[i].Package < res.Needs[j].Package }) for _, p := range res.ReadsOutside { if m.Say != nil { - m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker) + m.Say("the %s level of %s always runs: its %s %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker, res.neverKept[p]) } } return res, nil @@ -316,18 +328,27 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he res.Needs = append(res.Needs, delivery.PackageNeeds{Package: path, Needs: n}) } } - if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil { - // The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random - // hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again. + declared, why, err := declaredReads(tree, root, res.hashed()) + if err != nil { + return fmt.Errorf("%s: its %s: %w", path, ReadsOutsideMarker, err) + } + if why != "" { + // The digest of a package whose reads outside itself are not listed is one nothing can have kept: the key is a + // fresh random hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never + // read again. var nonce [32]byte if _, err := rand.Read(nonce[:]); err != nil { return err } res.Packages[path] = hex.EncodeToString(nonce[:]) res.ReadsOutside = append(res.ReadsOutside, path) + if res.neverKept == nil { + res.neverKept = map[string]string{} + } + res.neverKept[path] = why continue } - d, err := packageDigest(root, testMain, pkgs, tree, header, beside) + d, err := packageDigest(root, testMain, pkgs, tree, header, beside, declared) if err != nil { return fmt.Errorf("%s: %w", path, err) } @@ -340,14 +361,15 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he // tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of // every module outside it, and the package's own test data. The beside lines (the siblings' commits) join the header // only when a file the tests reach names MESH_CHECK_BESIDE. A package of the standard library is the Go version's. -func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside []string) (string, error) { +// The lines a package's ReadsOutsideMarker declares join them (declaredReads). +func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside, declared []string) (string, error) { reached := map[string]*goPackage{root.ImportPath: root} for _, dep := range testMain.Deps { if p := pkgs[dep]; p != nil { reached[dep] = p } } - lines := []string{} + lines := append([]string{}, declared...) readsBeside := false for _, p := range reached { if p.Standard || p.Module == nil { @@ -469,3 +491,107 @@ func contentLine(tree string, p *goPackage, path string) (string, error) { sum := sha256.Sum256(raw) return "file " + label + " " + hex.EncodeToString(sum[:]), nil } + +// hashed is the cache of content lines a digest step shares across its packages. +func (r *DigestResult) hashed() map[string]string { + if r.hashedFiles == nil { + r.hashedFiles = map[string]string{} + } + return r.hashedFiles +} + +// declaredReads are the digest lines a package's ReadsOutsideMarker adds: each of its lines, the content line of every +// file it names, and those of every file of the package's own directory. Without a marker it answers nothing. why is +// set, and the lines nil, when the marker cannot be trusted: it lists nothing, a line is not a clean relative path, or +// a line without ? matches nothing. +func declaredReads(tree string, root *goPackage, cache map[string]string) (lines []string, why string, err error) { + raw, err := os.ReadFile(filepath.Join(root.Dir, ReadsOutsideMarker)) + if errors.Is(err, fs.ErrNotExist) { + return nil, "", nil + } + if err != nil { + return nil, "", err + } + var patterns []string + for _, l := range strings.Split(string(raw), "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { + patterns = append(patterns, l) + } + } + if len(patterns) == 0 { + return nil, "lists nothing", nil + } + seen := map[string]bool{} + add := func(path string) error { + if seen[path] { + return nil + } + seen[path] = true + l, ok := cache[path] + if !ok { + var err error + if l, err = contentLine(tree, root, path); err != nil { + return err + } + cache[path] = l + } + lines = append(lines, "read "+l) + return nil + } + for _, l := range patterns { + pattern, absent := strings.CutPrefix(l, "?") + if !cleanTreePattern(pattern) { + return nil, fmt.Sprintf("names %q, which is not a clean path from the repository's root", l), nil + } + matches, err := filepath.Glob(filepath.Join(tree, filepath.FromSlash(pattern))) + if err != nil { + return nil, fmt.Sprintf("names %q, which is not a glob", l), nil + } + if len(matches) == 0 && !absent { + return nil, fmt.Sprintf("names %q, which matches nothing (a path whose absence is read starts with ?)", l), nil + } + lines = append(lines, "reads "+l) + for _, match := range matches { + if err := walkFiles(match, add); err != nil { + return nil, "", err + } + } + } + // The package's own files that are not Go source: its lists, its marker. A catalogue module's are its module's too. + err = walkFiles(root.Dir, func(path string) error { + if strings.HasSuffix(path, ".go") { + return nil + } + return add(path) + }) + return lines, "", err +} + +// cleanTreePattern says a marker's line is a path from the repository's root that stays inside it. +func cleanTreePattern(p string) bool { + if p == "" || strings.ContainsAny(p, " \t\\") || strings.HasPrefix(p, "/") || path.Clean(p) != p { + return false + } + for _, e := range strings.Split(p, "/") { + if e == ".." || e == "." { + return false + } + } + return true +} + +// walkFiles calls each with every file at or under path, skipping .git and node_modules, in lexical order. +func walkFiles(path string, each func(string) error) error { + return filepath.WalkDir(path, func(p string, e fs.DirEntry, err error) error { + if err != nil { + return err + } + if e.IsDir() { + if n := e.Name(); p != path && (n == ".git" || n == "node_modules") { + return filepath.SkipDir + } + return nil + } + return each(p) + }) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index a2bdd155..1101191d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -173,7 +173,7 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}} pkgs[root.ImportPath] = root digest := func() string { - d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil) + d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil, nil) if err != nil { t.Fatal(err) } @@ -190,28 +190,73 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { } } -// Issue 1935: a package whose directory carries the reads-outside marker is never kept: its digest is one no kept verdict -// can have (it differs on every call, though nothing changed), and the answer names it. A package without the marker -// keeps its digest. -func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) { +// Issue 1935, 1936: a package whose reads-outside marker lists nothing it can trust is never kept: its digest is one no +// kept verdict can have (it differs on every call, though nothing changed), and the answer names it. A marker that lists +// what the tests read is kept: a change to a file it names changes the package's digest, a change elsewhere does not. +func TestAPackageWithTheReadsOutsideMarkerIsKeptOnlyByWhatItLists(t *testing.T) { t.Parallel() m, tree := digestFixture(t) plain := digests(t, m, "unit") if len(plain.ReadsOutside) != 0 { t.Fatalf("a package carries the marker in the fixture: %v", plain.ReadsOutside) } - edit(t, tree, "c/"+ReadsOutsideMarker, "the tests read ../docs\n") - first, second := digests(t, m, "unit"), digests(t, m, "unit") - c := "fixture/digest/c" - if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] { - t.Errorf("c carries the marker and its digest is stable: %s, %s, before %s", first.Packages[c], second.Packages[c], plain.Packages[c]) + c, a := "fixture/digest/c", "fixture/digest/a" + if err := os.MkdirAll(filepath.Join(tree, "docs"), 0o755); err != nil { + t.Fatal(err) } - if strings.Join(first.ReadsOutside, ",") != c { - t.Errorf("the answer does not name the package that reads outside: %v", first.ReadsOutside) + edit(t, tree, "docs/one.md", "one\n") + edit(t, tree, "other.md", "other\n") + for name, marker := range map[string]string{ + "an empty marker": "# nothing listed\n\n", + "a sentence": "the tests read ../docs\n", + "a path climbing out": "../docs\n", + "a path matching nothing": "docs/none.md\n", + "an absolute path": "/etc/passwd\n", + "a path that is not clean": "docs/./one.md\n", + "a good line and a bad line": "docs\nnope/*.md\n", + } { + edit(t, tree, "c/"+ReadsOutsideMarker, marker) + first, second := digests(t, m, "unit"), digests(t, m, "unit") + if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] { + t.Errorf("%s: c's digest is stable: %s, %s, before %s", name, first.Packages[c], second.Packages[c], plain.Packages[c]) + } + if strings.Join(first.ReadsOutside, ",") != c { + t.Errorf("%s: the answer does not name the package that reads outside: %v", name, first.ReadsOutside) + } + if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] { + t.Errorf("%s: a has no marker and its digest moved: %s, %s, before %s", name, first.Packages[a], second.Packages[a], plain.Packages[a]) + } } - a := "fixture/digest/a" - if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] { - t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a]) + + // A list: a glob, a directory read whole, and a path whose absence is read. + edit(t, tree, "c/"+ReadsOutsideMarker, "# what c's tests read\ndocs/*.md\ndocs\n?gone\n") + before := digests(t, m, "unit") + if len(before.ReadsOutside) != 0 || len(before.Packages[c]) != 64 || before.Packages[c] == plain.Packages[c] { + t.Fatalf("a listed marker is not kept: %v, %s (no marker %s)", before.ReadsOutside, before.Packages[c], plain.Packages[c]) + } + if again := digests(t, m, "unit"); again.Packages[c] != before.Packages[c] { + t.Errorf("nothing changed and c's digest moved") + } + for _, step := range []struct { + file, text string + moves bool + }{ + {"other.md", "changed\n", false}, // outside the list + {"d/d.go", "package d\n\n// D.\n", false}, // a package c's tests do not reach + {"docs/one.md", "changed\n", true}, // a listed file + {"docs/two.md", "new\n", true}, // a new file the glob matches + {"gone", "here\n", true}, // a path whose absence was read + {"c/note.txt", "changed\n", true}, // the package's own file that is not Go + } { + edit(t, tree, step.file, step.text) + after := digests(t, m, "unit") + if moved := after.Packages[c] != before.Packages[c]; moved != step.moves { + t.Errorf("a change to %s: c's digest moved %v, want %v", step.file, moved, step.moves) + } + if after.Packages[a] != plain.Packages[a] { + t.Errorf("a change to %s moved a's digest", step.file) + } + before = after } // Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/reads-outside b/modules/mesh-build-go/cmd/mesh-build-go/reads-outside index c1b869c0..ea239b40 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/reads-outside +++ b/modules/mesh-build-go/cmd/mesh-build-go/reads-outside @@ -1 +1,7 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the manifests through mesh-sdk's validator (the predefined keys), gitea's and mesh-build-node's. +predefined.json +modules/gitea +modules/mesh-build-node diff --git a/modules/mesh-build-image/cmd/mesh-build-image/reads-outside b/modules/mesh-build-image/cmd/mesh-build-image/reads-outside index c1b869c0..b79eb772 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/reads-outside +++ b/modules/mesh-build-image/cmd/mesh-build-image/reads-outside @@ -1 +1,10 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the manifests through mesh-sdk's validator (the predefined keys), with the modules they compose with. +predefined.json +modules/gitea +modules/mesh-build-go +modules/mesh-build-node +modules/buildah +modules/docker diff --git a/modules/mesh-build-node/cmd/mesh-build-node/reads-outside b/modules/mesh-build-node/cmd/mesh-build-node/reads-outside index c1b869c0..26937843 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/reads-outside +++ b/modules/mesh-build-node/cmd/mesh-build-node/reads-outside @@ -1 +1,5 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# its manifest through mesh-sdk's validator, which reads the root's predefined keys. +predefined.json diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside index f6b2d871..8deabea3 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest: the gate is the seat's only claimant. +modules/*/module.json diff --git a/modules/mesh-issues/cmd/mesh-issues/reads-outside b/modules/mesh-issues/cmd/mesh-issues/reads-outside index f6b2d871..628ce47b 100644 --- a/modules/mesh-issues/cmd/mesh-issues/reads-outside +++ b/modules/mesh-issues/cmd/mesh-issues/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every module folder: its manifest, and whether it holds code (go.mod, package.json). +modules/*/module.json +modules/*/go.mod +modules/*/package.json diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside b/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside index f6b2d871..8ac753d2 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the root's retired lines. +retired-lines diff --git a/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside b/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside index f6b2d871..828cd198 100644 --- a/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside +++ b/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# mesh-mail's mailevent.go, of which this package holds a copy. +modules/mesh-mail/cmd/mesh-mail/mailevent.go diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside b/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside index f6b2d871..828cd198 100644 --- a/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# mesh-mail's mailevent.go, of which this package holds a copy. +modules/mesh-mail/cmd/mesh-mail/mailevent.go diff --git a/modules/mesh-mail/cmd/mesh-mail/reads-outside b/modules/mesh-mail/cmd/mesh-mail/reads-outside index f6b2d871..9088f24e 100644 --- a/modules/mesh-mail/cmd/mesh-mail/reads-outside +++ b/modules/mesh-mail/cmd/mesh-mail/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# messenger's content rule, of which content_rule.go is a copy. +modules/messenger/cmd/messenger/content.go diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside index c1b869c0..5fba8c24 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside @@ -1 +1,7 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest and the templates the moved modules name (in files/, or templates/). +modules/*/module.json +modules/*/files +modules/*/templates diff --git a/modules/minio/cmd/object-store/reads-outside b/modules/minio/cmd/object-store/reads-outside index c1b869c0..756af267 100644 --- a/modules/minio/cmd/object-store/reads-outside +++ b/modules/minio/cmd/object-store/reads-outside @@ -1 +1,5 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# brother's store.json, against the object store's. +modules/brother/files/store.json diff --git a/modules/networkmanager/cmd/uplink-tools/reads-outside b/modules/networkmanager/cmd/uplink-tools/reads-outside index f6b2d871..ceb6d7da 100644 --- a/modules/networkmanager/cmd/uplink-tools/reads-outside +++ b/modules/networkmanager/cmd/uplink-tools/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of uplink-tools in both modules that hold the uplink. +modules/networkmanager/cmd/uplink-tools/*.go +modules/systemd-networkd/cmd/uplink-tools/*.go diff --git a/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside b/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside +++ b/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/nm-applet/cmd/nm-applet-tools/reads-outside b/modules/nm-applet/cmd/nm-applet-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/nm-applet/cmd/nm-applet-tools/reads-outside +++ b/modules/nm-applet/cmd/nm-applet-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/openrazer/cmd/openrazer-tools/reads-outside b/modules/openrazer/cmd/openrazer-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/openrazer/cmd/openrazer-tools/reads-outside +++ b/modules/openrazer/cmd/openrazer-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/pacman/cmd/pacman-tools/reads-outside b/modules/pacman/cmd/pacman-tools/reads-outside index 2c69a776..856f5874 100644 --- a/modules/pacman/cmd/pacman-tools/reads-outside +++ b/modules/pacman/cmd/pacman-tools/reads-outside @@ -1 +1,5 @@ -These tests read the repository's root predefined.json, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the root's predefined keys. +predefined.json diff --git a/modules/polychromatic/cmd/polychromatic-tools/reads-outside b/modules/polychromatic/cmd/polychromatic-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/polychromatic/cmd/polychromatic-tools/reads-outside +++ b/modules/polychromatic/cmd/polychromatic-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/postgres/cmd/postgres-provider/reads-outside b/modules/postgres/cmd/postgres-provider/reads-outside index c1b869c0..1aaa7f89 100644 --- a/modules/postgres/cmd/postgres-provider/reads-outside +++ b/modules/postgres/cmd/postgres-provider/reads-outside @@ -1 +1,5 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest. +modules/*/module.json diff --git a/modules/slack/cmd/slack-tools/reads-outside b/modules/slack/cmd/slack-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/slack/cmd/slack-tools/reads-outside +++ b/modules/slack/cmd/slack-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/systemd-networkd/cmd/uplink-tools/reads-outside b/modules/systemd-networkd/cmd/uplink-tools/reads-outside index f6b2d871..ceb6d7da 100644 --- a/modules/systemd-networkd/cmd/uplink-tools/reads-outside +++ b/modules/systemd-networkd/cmd/uplink-tools/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of uplink-tools in both modules that hold the uplink. +modules/networkmanager/cmd/uplink-tools/*.go +modules/systemd-networkd/cmd/uplink-tools/*.go diff --git a/modules/xorg/internal/desktop/reads-outside b/modules/xorg/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/xorg/internal/desktop/reads-outside +++ b/modules/xorg/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/xterm/internal/desktop/reads-outside b/modules/xterm/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/xterm/internal/desktop/reads-outside +++ b/modules/xterm/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go From c0859751461aa112c60a163b25b5694cfd402b06 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:05:42 +0100 Subject: [PATCH 22/28] Pin mesh-sdk 9796444 (l2-next: feat/1126-nodes-compose merged with sdk main #101, the standing-test race fixes) (issue 1126) --- checks/placement/go.mod | 2 +- checks/placement/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/checks/placement/go.mod b/checks/placement/go.mod index 902a02d4..6085d760 100644 --- a/checks/placement/go.mod +++ b/checks/placement/go.mod @@ -2,4 +2,4 @@ module checks/placement go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026205647-5f45175ed02b +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 diff --git a/checks/placement/go.sum b/checks/placement/go.sum index 18b1e59c..05a0bf8b 100644 --- a/checks/placement/go.sum +++ b/checks/placement/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026205647-5f45175ed02b h1:6iRVEZi6HUi9r2pFPz8aMLJy6FPdnyTOZ79/a7eSbNI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026205647-5f45175ed02b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= From 234d2d55343ba2af2f03a66944598fcee37f10ff Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:11:55 +0100 Subject: [PATCH 23/28] mesh-delivery-gate: pin mesh-sdk b4310df, the delivery-records work merged with sdk main (issue 1280) --- modules/mesh-delivery-gate/go.mod | 2 +- modules/mesh-delivery-gate/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 1a02ef85..c5acb0be 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026201248-e44cbaf57fc3 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210442-b4310dfb8ec6 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index 73a8b6b4..64b301b0 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026201248-e44cbaf57fc3 h1:kOAR8GaTpOP9j2XNm3grqEfSwSDHnWbHKd+Qftkr/QE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026201248-e44cbaf57fc3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210442-b4310dfb8ec6 h1:JDgD6KA80fzQzVj/NFQD7hXVsZQfsYW6xjZp0MoPDUU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210442-b4310dfb8ec6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From d224b74ff1b942a9fe504a2d54bad15601eed2d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:13:01 +0100 Subject: [PATCH 24/28] mesh-build-go: a marked package's digest hashes every file of its own directory, Go files its build tags exclude too (issue 1936) go list leaves a file its build tags exclude (an integration test) in IgnoredGoFiles, which the digest did not hash; mesh-provisioner-client's unit test globs *_test.go and reads two of them, so a change there reused a stale unit pass. Review finding F2 on ba49a7f0. --- .../mesh-build-go/cmd/mesh-build-go/digest.go | 18 ++++++++---------- .../cmd/mesh-build-go/digest_test.go | 13 +++++++------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index ae275800..0e0133e2 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -44,9 +44,10 @@ const besideVariable = "MESH_CHECK_BESIDE" // with ? names a path whose absence is what the tests read (a file that must not be there): it may match nothing. // // The digest then hashes, besides what the toolchain lists, every line of the marker, every file it names (under its -// path in the tree, by content) and every file of the package's own directory. A marker that lists nothing, a line -// that is not a clean relative path, or a line without ? that matches nothing makes the package one that is never kept -// (its digest a fresh random hex): a list nobody can trust keeps nothing. The exact fix, the files a test opened kept +// path in the tree, by content) and every file of the package's own directory, Go files included (a file its build tags +// exclude is one go list does not hash). A marker that lists nothing, a line that is not a clean relative path, or a +// line without ? that matches nothing makes the package one that is never kept (its digest a fresh random hex): a list +// nobody can trust keeps nothing. The exact fix, the files a test opened kept // from the Go test log with each verdict, replaces the lists (issue 1936). const ReadsOutsideMarker = "reads-outside" @@ -557,13 +558,10 @@ func declaredReads(tree string, root *goPackage, cache map[string]string) (lines } } } - // The package's own files that are not Go source: its lists, its marker. A catalogue module's are its module's too. - err = walkFiles(root.Dir, func(path string) error { - if strings.HasSuffix(path, ".go") { - return nil - } - return add(path) - }) + // Every file of the package's own directory: its lists, its marker, and its Go files too, since go list leaves a + // file its build tags exclude (an integration test a unit test globs) out of what the digest hashes. A file the + // toolchain lists is hashed twice, which changes nothing. + err = walkFiles(root.Dir, add) return lines, "", err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index 1101191d..a6dac3a4 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -241,12 +241,13 @@ func TestAPackageWithTheReadsOutsideMarkerIsKeptOnlyByWhatItLists(t *testing.T) file, text string moves bool }{ - {"other.md", "changed\n", false}, // outside the list - {"d/d.go", "package d\n\n// D.\n", false}, // a package c's tests do not reach - {"docs/one.md", "changed\n", true}, // a listed file - {"docs/two.md", "new\n", true}, // a new file the glob matches - {"gone", "here\n", true}, // a path whose absence was read - {"c/note.txt", "changed\n", true}, // the package's own file that is not Go + {"other.md", "changed\n", false}, // outside the list + {"d/d.go", "package d\n\n// D.\n", false}, // a package c's tests do not reach + {"docs/one.md", "changed\n", true}, // a listed file + {"docs/two.md", "new\n", true}, // a new file the glob matches + {"gone", "here\n", true}, // a path whose absence was read + {"c/note.txt", "changed\n", true}, // the package's own file that is not Go + {"c/x_integration_test.go", "//go:build integration\n\npackage c\n", true}, // a Go file its build tags leave out (go list's IgnoredGoFiles) } { edit(t, tree, step.file, step.text) after := digests(t, m, "unit") From d58249d0d80db7e0af80caa50d2f6004264cc8c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:13:01 +0100 Subject: [PATCH 25/28] checks/health: its marker lists the predefined keys, the modules' templates and health-undeclared it reads (issue 1936) ReadTreeManifest reads predefined.json and each templated module's templates (WithTemplatesFrom), and the test reads the root's health-undeclared through Kept: none was listed, so a change to them reused a stale verdict. No other marker's package reads manifests through ReadTreeManifest beyond what it lists. Review finding F1 on ba49a7f0. --- checks/health/reads-outside | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/checks/health/reads-outside b/checks/health/reads-outside index 1aaa7f89..419af578 100644 --- a/checks/health/reads-outside +++ b/checks/health/reads-outside @@ -1,5 +1,10 @@ # What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; # ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept # until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. -# every manifest. +# every manifest through mesh-sdk's ReadTreeManifest: the root's predefined keys and each templated module's +# templates (files/ by default, or the directory its manifest names); and the root's kept count of undeclared ports. modules/*/module.json +predefined.json +modules/*/files +modules/*/templates +health-undeclared From 488b7bfc58167ac16286971664cebffb24dd87ba Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:13:14 +0100 Subject: [PATCH 26/28] checks/health: its marker names health-undeclared for what it counts (issue 1936) --- checks/health/reads-outside | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/health/reads-outside b/checks/health/reads-outside index 419af578..f6ce1405 100644 --- a/checks/health/reads-outside +++ b/checks/health/reads-outside @@ -2,7 +2,7 @@ # ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept # until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. # every manifest through mesh-sdk's ReadTreeManifest: the root's predefined keys and each templated module's -# templates (files/ by default, or the directory its manifest names); and the root's kept count of undeclared ports. +# templates (files/ by default, or the directory its manifest names); and the root's kept count of resources that do not say how they are ready. modules/*/module.json predefined.json modules/*/files From 099605ec70a26db5ae8cc9cb1959ec0ce7a0c060 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:18:08 +0100 Subject: [PATCH 27/28] mesh-delivery-gate: the Approve's ask says the check's actual state at its heads, never a pass it has not had (issue 1733) With check-gates-merge false a head is ready and asked before any verdict exists, yet the ask said " passed its checks" (a group: "passed its checks"): on 2026-10-26 the operator approved mesh-delivery #79 and a six-repository group on that claim while their checks had not answered. The ask now reads the checker's verdict at exactly the heads it covers (checkAt, beside checkerAt): "passed its check" only for a pass there; otherwise "its check has no verdict yet[: it is running (since HH:MM UTC)]" or "its check failed: ", and the Approve clause says it merges "now, before its check ends" or "although its check failed". While the check gates no merge, a pending ask whose check then passes or fails (or loses a pass it named) is withdrawn and asked again in new words (the ask's AskCheck against the state now, endStaleAsks, the review-record path's withdrawAsk); a check that starts running is no new word, so the wait stays one message. Whether a merge waits is unchanged. The gate's log lines and stalled lines that claimed a pass for a ready head say the same state. Quick fix: replace "passed its checks" by neutral words in the two summaries. Thorough fix: one reader of the check's state at the ask's heads (checkAt) used by every text that names it, and the ask re-said when that state changes. Chosen: thorough, as it costs one helper and one case in endStaleAsks, and the quick one would still leave a pending ask silent about a failure that came after it. Tests: rows in TestAnApprovedPullRequestIsMergedThroughTheMergeTool (gated pass; ungated no verdict, running, then passed, then failed) and the ungated group's wording; no new test function. --- .../cmd/mesh-delivery-gate/approval.go | 63 ++++++++++++--- .../cmd/mesh-delivery-gate/merges.go | 2 +- .../cmd/mesh-delivery-gate/onestatus.go | 69 +++++++++++++++++ .../cmd/mesh-delivery-gate/ready.go | 23 ++++-- .../cmd/mesh-delivery-gate/ready_test.go | 76 +++++++++++++++++++ 5 files changed, 214 insertions(+), 19 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 3c385094..11bd9e02 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -148,8 +148,32 @@ func touchesSaid(d *Delivery) string { return strings.Join(d.Check.Modules, ", ") } +// checkClause is the check's state as the ask's summary says it after the pull request's or the group's name: sep and +// "passed its check", or ": its check has no verdict yet: it is running (since 21:05 UTC)" and the like. +func checkClause(c checkSaid, sep string) string { + if c.passed() { + return sep + c.Words + } + return ": " + c.Words +} + +// approveCaveat is what an Approve does beyond the merge while the check has not passed: it merges before the check ends, +// or although it failed (the check gates no merge, check-gates-merge false). Empty once it passed. +func approveCaveat(c checkSaid) string { + switch c.Key { + case checkSaidPass: + return "" + case checkSaidFail: + return " although its check failed" + } + return " now, before its check ends" +} + // composeMergeAsk is the ask for a delivery, or for a group's members together. -func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, now time.Time) asks.Ask { +// check is the state of the check at the heads it covers (checkAt): the ask says "passed its check" only when it did, and +// otherwise that Approve merges before the check ends, or although it failed (issue 1733). +func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, check checkSaid, + now time.Time) asks.Ask { var a asks.Ask a.ID, a.Who, a.Expires = id, asks.Operator, now.Add(AskLasts) a.OnExpiry = "nothing is merged; the ask is made again" @@ -170,8 +194,8 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, } if group == "" { a.Headline = phone("Merge into main: "+pullSaid(d)+" "+d.Title, asks.HeadlineLength) - summary := fmt.Sprintf("%s %s passed its checks. Approve merges it into main at %s; Decline leaves it unmerged.", - pullSaid(d), d.Title, shortOf(d.Commit, 12)) + summary := fmt.Sprintf("%s %s%s. Approve merges it into main at %s%s; Decline leaves it unmerged.", + pullSaid(d), d.Title, checkClause(check, " "), shortOf(d.Commit, 12), approveCaveat(check)) if unreviewed != "" { summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary } @@ -187,8 +211,12 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, names = append(names, pullSaid(m)) } a.Headline = phone("Merge into main: group "+group+" ("+strings.Join(names, ", ")+")", asks.HeadlineLength) - summary := fmt.Sprintf("The group %s, %d pull requests, passed its checks. Approve merges all of them into main in one pass, at the heads named; Decline leaves them unmerged.", - group, len(members)) + caveat := approveCaveat(check) + if caveat != "" { + caveat = "," + caveat + } + summary := fmt.Sprintf("The group %s, %d pull requests%s. Approve merges all of them into main in one pass, at the heads named%s; Decline leaves them unmerged.", + group, len(members), checkClause(check, ", "), caveat) if unreviewed != "" { summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary } @@ -309,6 +337,10 @@ func (h *Holder) endStaleAsks() { h.cancels = append(h.cancels, h.dropAsk(r.AskID, "its head "+shortOf(d.Commit, 12)+" is no longer the pull request's head, or the pull request closed")) case !h.headsHold(d, r): h.cancels = append(h.cancels, h.dropAsk(r.AskID, "a head it covers moved on: the heads it covers are no longer the group's heads")) + case r.Approved == nil && r.AskID != "" && r.AskCheck != "" && !h.checkGates() && h.checkMoved(d, r.AskCheck): + // The check gates no merge, so the ask was made before its verdict: once the check passes or fails (or a pass it + // named is gone), the ask is asked again in words that say so (issue 1733). Running and not started say the same. + h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, "the check at its heads is now: "+h.checkAt(h.askMembers(d)).Words)) case r.Approved == nil && r.AskID != "" && !r.AskedAt.IsZero() && !h.Now().Before(r.AskedAt.Add(AskLasts)): // The ask lapsed at the router with nobody's answer, and nothing in the gate's inputs says so: it is taken away and // written again, which the teller asks afresh (ADR 0441 §1). @@ -317,6 +349,16 @@ func (h *Holder) endStaleAsks() { } } +// checkMoved is whether the check at the heads of d's ask reached a verdict its pending ask does not say: it passed or failed +// since, or the pass it said is gone. Under the lock. +func (h *Holder) checkMoved(d *Delivery, said string) bool { + now := h.checkAt(h.askMembers(d)).Key + if now == said { + return false + } + return now == checkSaidPass || now == checkSaidFail || said == checkSaidPass || said == checkSaidFail +} + // approval is what a delivery that may merge waits on for the operator: empty once its head is approved, and the ask // to make when none is made yet. Under the lock. func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { @@ -367,19 +409,20 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { asking = max(asking, m.Ready.Asking+1) } } - ask := composeMergeAsk(sdk.AskingID(id, asking), d, group, members, heads, now) + check := h.checkAt(members) + ask := composeMergeAsk(sdk.AskingID(id, asking), d, group, members, heads, check, now) taken := []*Delivery{} var reads []memberRead for _, m := range members { h.runOf(m, now) m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) - m.Ready.AskedKey, m.Ready.Asking = id, asking + m.Ready.AskedKey, m.Ready.Asking, m.Ready.AskCheck = id, asking, check.Key h.keep(m) taken = append(taken, m) reads = append(reads, readOf(m, m.Commit)) } - h.Logf("[mesh-delivery] %s passed its checks: the operator is asked to Approve its merge into main (%s, ask %d)", - d.ID, id, r.Asks) + h.Logf("[mesh-delivery] %s (%s): the operator is asked to Approve its merge into main (%s, ask %d)", + d.ID, check.Words, id, r.Asks) gid := "" if group != "" { gid = d.Group @@ -432,7 +475,7 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { h.mu.Lock() defer h.mu.Unlock() undo() - h.Logf("[mesh-delivery] %s passed its checks and the operator is not asked: %s", d.ID, says) + h.Logf("[mesh-delivery] %s (%s): the operator is not asked: %s", d.ID, check.Words, says) h.lookAt(h.Now(), "an ask not made: the forge account cannot write a repository it covers") return } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index b38e721f..a3055a32 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -287,7 +287,7 @@ func (h *Holder) ForgeMerged(f sdk.Forge) { r, now := d.Ready, h.Now() d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: "merged", Line: string(mustJSON(mergedEvent(d, f.Head))), Since: now}) r.Merged, r.Answer, r.RefusedSince, r.merging, r.MergeCommit = &now, "merged", nil, false, f.Merge - h.Logf("[mesh-delivery] %s merged on the operator's Approve at %s, its check passed against its trunk's head", d.ID, shortOf(f.Head, 12)) + h.Logf("[mesh-delivery] %s merged on the operator's Approve at %s (%s)", d.ID, shortOf(f.Head, 12), h.checkAt(h.askMembers(d)).Words) h.keep(d) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go index 10151f1f..76386e98 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go @@ -302,3 +302,72 @@ func checkerRefusal(id string) error { "its trunk or its group, and nothing is asked of it here (novox/hq ADR 0447). Read where its check stands with the "+ "checker's checker_pull tool", id) } + +// The states of the check at an ask's heads, as the ask says them (issue 1733). +const ( + checkSaidPass = "pass" + checkSaidRunning = "running" + checkSaidNone = "none" + checkSaidFail = "fail" +) + +// checkSaid is the state of the check at the heads an ask covers, and its words for the ask: what the checker's verdict at +// exactly those heads says, never what the gate would like it to say (issue 1733). Key is one of the checkSaid states. +type checkSaid struct { + Key string + Words string +} + +// passed is whether the check at the heads passed. +func (c checkSaid) passed() bool { return c.Key == checkSaidPass } + +// checkAt reads the check at these members' heads: the checker's verdict for exactly those heads (its verdicts bucket, +// keyed by their hash), and, while none is held, the forge's mesh/check at a member's head (a failure or a pending one +// there). "passed its check" is said only for a passing verdict at these heads. Under the lock. +func (h *Holder) checkAt(members []*Delivery) checkSaid { + v, hash, ok := h.checkerAt(members) + if ok { + switch v.State { + case checkerPass: + return checkSaid{Key: checkSaidPass, Words: "passed its check"} + case checkerRunning: + words := "its check has no verdict yet: it is running" + if since, held := h.runningSince[hash]; held && !since.IsZero() { + words += " (since " + since.UTC().Format("15:04") + " UTC)" + } + return checkSaid{Key: checkSaidRunning, Words: words} + default: + why := firstOf(oneLine(v.Failed.Test), oneLine(v.Failed.Words), v.State) + if v.Failed.Test != "" && v.Failed.Words != "" { + why = oneLine(v.Failed.Test) + ": " + oneLine(v.Failed.Words) + } + return checkSaid{Key: checkSaidFail, Words: "its check failed: " + clip(why, 120)} + } + } + for _, m := range members { + switch m.CheckState { + case "failure", "error": + why := "mesh/check is " + m.CheckState + " at " + shortOf(m.Commit, 12) + if m.Check != nil && m.Check.Summary != "" { + why = oneLine(m.Check.Summary) + } + return checkSaid{Key: checkSaidFail, Words: "its check failed: " + clip(why, 120)} + } + } + succeeded := 0 + for _, m := range members { + switch m.CheckState { + case "pending": + return checkSaid{Key: checkSaidRunning, Words: "its check has no verdict yet: it is running"} + case "success": + succeeded++ + } + } + switch { + case succeeded > 0 && len(members) == 1: + return checkSaid{Key: checkSaidPass, Words: "passed its check"} // its head's mesh/check is the checker's verdict at it + case succeeded == len(members) && succeeded > 0: + return checkSaid{Key: checkSaidNone, Words: "its members' own checks passed; the check of their heads together has not answered"} + } + return checkSaid{Key: checkSaidNone, Words: "its check has no verdict yet"} +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 4b96bc11..aee84748 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -115,6 +115,10 @@ type ReadyRun struct { Heads map[string]string `json:"heads,omitempty"` AskedAt time.Time `json:"asked_at"` Asks int `json:"asks,omitempty"` + // AskCheck is the state of the check at the heads the pending ask names, as its words said it (checkAt: pass, running, + // none or fail; issue 1733): while the check gates no merge, an ask whose check reached a verdict since is asked again + // in new words, so it never keeps a claim the check no longer holds. + AskCheck string `json:"ask_check,omitempty"` // Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused // why an answer was not taken (it did not hold to the ask). Approved *time.Time `json:"approved,omitempty"` @@ -186,7 +190,8 @@ func (h *Holder) readyAsks(now time.Time) []func() { if d.Ready == nil && d.State == Ready { h.runOf(d, now) h.Poke() // its check, if none is asked, is worth asking now (askCheck) - h.Logf("[mesh-delivery] %s: its checks passed; the operator's Approve of its merge into main is asked once it may merge", d.ID) + h.Logf("[mesh-delivery] %s is ready (%s); the operator's Approve of its merge into main is asked once it may merge", + d.ID, h.checkAt([]*Delivery{d}).Words) } if a := h.sayAfterLine(d); a != nil { asks = append(asks, a) @@ -227,11 +232,12 @@ func (h *Holder) readyAsks(now time.Time) []func() { if d.Base != MergeBranch { if r.OtherBase != d.Base { r.OtherBase = d.Base - h.Logf("[mesh-delivery] %s passed its checks and is not asked: it merges into %s, not %s", d.ID, d.Base, MergeBranch) + check := h.checkAt([]*Delivery{d}).Words // ready need not mean passed: the check may gate no merge (issue 1733) + h.Logf("[mesh-delivery] %s (%s) is not asked: it merges into %s, not %s", d.ID, check, d.Base, MergeBranch) h.keep(d) - asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request's checks passed at "+ - "%s, and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+ - "author's (novox/hq ADR 0398 §4).", d.Commit, d.Base, MergeBranch))) + asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request is at %s (%s), "+ + "and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+ + "author's (novox/hq ADR 0398 §4).", d.Commit, check, d.Base, MergeBranch))) } continue } @@ -524,13 +530,14 @@ func (h *Holder) readyLines(now time.Time) []StalledLine { case d.Base == "" && d.Number != 0 && now.Sub(r.Seen) >= headHeardBound: out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(), Bound: headHeardBound.String(), H2: "none: a later announcement of the pull request's head (the merge tool announces it again, the forge's announcer restarts, or a new push) or an edit of its description; a recheck does not fill it — the operator's", - Says: fmt.Sprintf("%s#%d passed its checks at %s and waits for %s", d.Repository, d.Number, shortOf(r.Head, 12), waitsForHead)}) + Says: fmt.Sprintf("%s#%d at %s (%s) waits for %s", d.Repository, d.Number, shortOf(r.Head, 12), + h.checkAt([]*Delivery{d}).Words, waitsForHead)}) case r.CannotWrite != "" && r.Approved == nil && r.CannotWriteSince != nil: out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(*r.CannotWriteSince).Round(time.Second).String(), Bound: "0s", H2: "none: the forge account given write on the repository (gitea_grant_write; the gitea module grants it itself once a module of it is registered), which asks the operator then", - Says: fmt.Sprintf("%s#%d passed its checks at %s and the operator is not asked for its Approve: %s", - d.Repository, d.Number, shortOf(r.Head, 12), r.CannotWrite)}) + Says: fmt.Sprintf("%s#%d at %s (%s): the operator is not asked for its Approve: %s", + d.Repository, d.Number, shortOf(r.Head, 12), h.checkAt([]*Delivery{d}).Words, r.CannotWrite)}) case r.Failed != "" && d.State == Rejected: // Its cure is its author's new head: listed here at once (`show`), told on the pull request at once, and the // operator's only once no new head came within the author's bound (novox/hq ADR 0505 §5). diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go index e15cc1e1..6e139b14 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go @@ -1,11 +1,15 @@ package main import ( + "encoding/json" "errors" "sort" "strings" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" ) // **A pull request whose checks passed and whose merge the operator approved is merged by this owner** (novox/hq issues @@ -65,6 +69,73 @@ func TestAnApprovedPullRequestIsMergedThroughTheMergeTool(t *testing.T) { t.Fatalf("merged again: %v", w.forge.merges) } + // 1733: the Approve's ask says the check's state at the heads it names, read from the checker's verdict there: "passed + // its check" only when it passed. While the check gates no merge, the ask is made before any verdict, says that Approve + // merges before the check ends, and is asked again in new words once the check passes or fails; a running check is no + // new word. + for _, row := range []struct { + name, state, failed string + gated bool + asks int + want, not []string + }{ + {name: "gated, passed", gated: true, state: "pass", asks: 1, + want: []string{"app#7 Fix it passed its check. Approve merges it into main at " + readyHead[:12] + ";"}, + not: []string{"before its check ends", "no verdict yet"}}, + {name: "ungated, no verdict", asks: 1, + want: []string{"its check has no verdict yet", "Approve merges it into main at " + readyHead[:12] + " now, before its check ends"}, + not: []string{"passed"}}, + {name: "ungated, running", state: "running", asks: 1, + want: []string{"its check has no verdict yet", "before its check ends"}, not: []string{"passed"}}, + {name: "ungated, then passed", state: "pass", asks: 2, + want: []string{"passed its check. Approve merges it into main at " + readyHead[:12] + ";"}, not: []string{"before its check ends"}}, + {name: "ungated, then failed", state: "fail", failed: "TestThing: wanted 1, got 2", asks: 2, + want: []string{"its check failed: TestThing: wanted 1, got 2", "although its check failed"}, not: []string{"passed"}}, + } { + t.Run(row.name, func(t *testing.T) { + v := newWorld(t) + if !row.gated { + ungated(v) + } + v.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Title = "Fix it" + v.h.PullUpdated(p) + put := func() { + hash, _ := sdk.HeadsHash([]sdk.Head{headOf("novox/app", 7, readyHead)}) + value, _ := json.Marshal(map[string]any{"v": 1, "group": "g", "heads": hash, "state": row.state, + "failed": map[string]string{"words": row.failed}}) + v.h.CheckerVerdictChanged("g."+hash, value) + } + if row.gated { + v.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + } + v.settleAll() + if !row.gated && row.state != "" { + put() + v.settleAll() + } + open := v.op.open() + if len(open) != 1 || v.op.count() != row.asks { + t.Fatalf("open asks %v, asked %d times; wanted one open, asked %d times", open, v.op.count(), row.asks) + } + e := v.op.ask(open[0]).Explanation + for _, w := range row.want { + if !strings.Contains(e, w) { + t.Fatalf("the ask does not say %q:\n%s", w, e) + } + } + for _, w := range row.not { + if strings.Contains(e, w) { + t.Fatalf("the ask says %q:\n%s", w, e) + } + } + if err := asks.Glance(e); err != nil { + t.Fatal(err) + } + }) + } + // check-gates-merge false (the operator, 2026-10-25 22:56): a reviewed pull request is asked with no verdict, another // merge moving its trunk puts it out of nothing, a failed verdict is shown on it and holds nothing, and the gate's // status and show say the mode. @@ -255,6 +326,11 @@ func TestAGroupMergesEveryMemberInOnePassOnlyOnceComposedAndApproved(t *testing. if u.op.count() != 1 { t.Fatalf("ungated, the group is asked %d times; wanted once", u.op.count()) } + // 1733: the group's ask claims no pass it has not had. + if e := u.op.ask(u.op.open()[0]).Explanation; strings.Contains(e, "passed") || + !strings.Contains(e, "2 pull requests: its check has no verdict yet. Approve merges all of them into main in one pass, at the heads named, now, before its check ends;") { + t.Fatalf("ungated, the group's ask does not say its check has no verdict yet:\n%s", e) + } u.op.answer(u.op.open()[0], OptionApprove) u.settleAll() if len(u.forge.merges) != 2 { From 4677b8a2f50189bcb5c28f39977ad484a74e1db5 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:21:21 +0100 Subject: [PATCH 28/28] mesh-delivery-gate: pin mesh-sdk 68d65e6, the group's sdk head merged with sdk main (issue 1280) --- modules/mesh-delivery-gate/go.mod | 2 +- modules/mesh-delivery-gate/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index c5acb0be..aee248e7 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210442-b4310dfb8ec6 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index 64b301b0..b3dab4f0 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210442-b4310dfb8ec6 h1:JDgD6KA80fzQzVj/NFQD7hXVsZQfsYW6xjZp0MoPDUU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210442-b4310dfb8ec6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=