diff --git a/.gitignore b/.gitignore index 18c2b03c..3a9d9d08 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ modules/mesh-issues/cmd/mesh-issues/mesh-issues modules/mounts/cmd/mounts/mounts modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge modules/mesh-knowledge/mesh-knowledge +modules/mesh-reads/mesh-reads +modules/mesh-reads/cmd/mesh-reads/mesh-reads diff --git a/modules/betterbird/README.md b/modules/betterbird/README.md index b86e73b9..082e2ec2 100644 --- a/modules/betterbird/README.md +++ b/modules/betterbird/README.md @@ -56,7 +56,15 @@ honours) and installs Owl into every profile from `file:///opt/betterbird/owl/ow `force_installed`, with `updates_disabled`. A profile that already runs another Owl keeps it: the policy installs the pinned one where Owl is missing. A newer Owl in the profile (g14 runs 1.5.7) is said as a fact by `betterbird_status`; Owl installed by policy cannot be removed in Betterbird, so the remedy is -moving the pin. An older one is a finding of `betterbird_check`, with the same remedy. **Owl's licence stays the person's**: the mesh holds no licence, key or payment (ADR 0316 §9). +moving the pin. An older one is a finding of `betterbird_check`, with the same remedy. + +**Betterbird never asks to be the default client.** The policy locks `mail.shell.checkDefaultClient` off +(`Preferences`, `Status: locked`): Betterbird's own default (`all-thunderbird.js`) is to ask at every start, +with a window to make it the default for mail, news, feeds and calendar files, while that preference is true +(`messenger.js`). Betterbird 153 has no policy of its own for it (its `policies-schema.json`), and its +`Preferences` policy accepts the `mail.` prefix. A lock, not a default: the window's own "always check" box +writes the preference into the profile, which wins over a default; a lock wins over the profile and greys the +box out in the settings. `betterbird_check` says when Betterbird would ask (novox/hq issue 345). **Owl's licence stays the person's**: the mesh holds no licence, key or payment (ADR 0316 §9). **No start of its own.** Betterbird is started by the operator from the launcher. The start (`/usr/local/bin/betterbird`) first runs `betterbird-tools prepare`, which, with no Betterbird of the @@ -159,7 +167,7 @@ and key. | `betterbird_user_js` (r; a with `apply`) | the declared user.js against the profile's; `apply` places it | | `betterbird_log` (r) | the last lines of the journal of `mesh-betterbird` (or of the reminder test), addresses and credentials masked (the value after `password=`, `token:` and the like, and the token after `Bearer`/`Basic`). A Betterbird the module did not start prints elsewhere, and the answer says where | | `betterbird_restart` (a) | ends Betterbird (asked to quit, forced after 8 s), places the user.js where allowed, starts it as `mesh-betterbird`. **Refused while a composer window is open** (X instance `Msgcompose`, whatever its class), because the unsent message would be lost, unless `discard_unsent` is true, which only the operator's yes justifies. It answers within the runtime's 30 s: a start not yet seen after 4 s is said in `note`, and `betterbird_status` shows it | -| `betterbird_check` (r) | installed from the mesh's release and not behind the newest published build; the policy placed; Owl at its pin and active; one running; profile found; the user.js on the machine, valid, placed, in effect; the declared accounts in the list, none refused, each signed in once; its output read | +| `betterbird_check` (r) | installed from the mesh's release and not behind the newest published build; the policy placed; no ask at the start to be the default mail client; Owl at its pin and active; one running; profile found; the user.js on the machine, valid, placed, in effect; the declared accounts in the list, none refused, each signed in once; its output read | | `betterbird_accounts` (r; a with `propose`) | the profile's accounts by count and kind (declarable, Owl's, others); with `propose`, the setting `accounts` built from them and proposed to the operator (ADR 0317 §9) | | `betterbird_reminder_test` (a) | a reminder on demand without the operator's calendar (below) | diff --git a/modules/betterbird/cmd/betterbird-tools/betterbird.go b/modules/betterbird/cmd/betterbird-tools/betterbird.go index 3759e49c..a84c73c1 100644 --- a/modules/betterbird/cmd/betterbird-tools/betterbird.go +++ b/modules/betterbird/cmd/betterbird-tools/betterbird.go @@ -849,7 +849,8 @@ type CheckAnswer struct { } // Check verifies what the module promises: Betterbird installed, one running in the session, its -// profile found, the module's user.js declared, valid, placed and in effect, and its output read. +// profile found, the module's user.js declared, valid, placed and in effect, its output read, and no ask at its +// start to be the default mail client (novox/hq issue 345). func (m *Machine) Check() (CheckAnswer, error) { a := CheckAnswer{Findings: []Finding{}} add := func(what, do string) { a.Findings = append(a.Findings, Finding{what, do}) } @@ -875,6 +876,10 @@ func (m *Machine) Check() (CheckAnswer, error) { if in.From == "the mesh" && !strings.HasPrefix(in.Policy, "placed:") { add("the enterprise policy is "+in.Policy, "push the module to this node") } + if why := m.defaultClientAsk(profilePath); why != "" && in.From != "not installed" { + add("Betterbird asks at its start to be the default mail client: "+why, + "push the module to this node: its policy locks "+defaultClientPref+" off (novox/hq issue 345)") + } switch { case in.From != "the mesh": case in.Owl == nil || in.Owl.Pinned == "": diff --git a/modules/betterbird/cmd/betterbird-tools/install.go b/modules/betterbird/cmd/betterbird-tools/install.go index 9e852c5b..1ec0204d 100644 --- a/modules/betterbird/cmd/betterbird-tools/install.go +++ b/modules/betterbird/cmd/betterbird-tools/install.go @@ -30,6 +30,10 @@ const ( // owlFile is Owl's xpi as the mesh placed it, and the URL the policy installs it from. owlFile = installDir + "/owl/owl.xpi" owlID = "owl@beonex.com" + // defaultClientPref is Betterbird's ask at its start to be the default mail, news, feed and calendar + // client (Thunderbird's all-thunderbird.js: pref("mail.shell.checkDefaultClient", true); messenger.js opens + // systemIntegrationDialog.xhtml while it is true). The module's policy locks it off (novox/hq issue 345). + defaultClientPref = "mail.shell.checkDefaultClient" // preparation is the tools' program as the module places it for its start (betterbird-tools prepare). preparation = "/usr/local/lib/mesh-betterbird/betterbird-tools" // versionsURL is where Betterbird publishes its newest build per major version: what its own updater @@ -167,7 +171,8 @@ func (m *Machine) policyState() string { if why := policyProblem(raw); why != "" { return "placed, and " + why } - return "placed: Betterbird's updater off, Owl installed into every profile from the mesh's xpi, its updates off" + return "placed: Betterbird's updater off, Owl installed into every profile from the mesh's xpi, its updates off, " + + "and the ask to be the default mail client locked off" } // policyProblem is what the policy fails to say of what ADR 0316 asks, or "". @@ -180,6 +185,10 @@ func policyProblem(raw []byte) string { URL string `json:"install_url"` Disabled bool `json:"updates_disabled"` } `json:"ExtensionSettings"` + Preferences map[string]struct { + Value any `json:"Value"` + Status string `json:"Status"` + } `json:"Preferences"` } `json:"policies"` } if err := json.Unmarshal(raw, &p); err != nil { @@ -196,9 +205,37 @@ func policyProblem(raw []byte) string { case !owl.Disabled: return "it leaves Owl's updates on" } + if pr, ok := p.Policies.Preferences[defaultClientPref]; !ok || pr.Value != false || pr.Status != "locked" { + return "it does not lock Betterbird's ask to be the default mail client off (Preferences " + defaultClientPref + ": false, locked)" + } return "" } +// defaultClientAsk says why Betterbird asks at its start to be the default mail client, or "" when it does +// not: the module's policy locks the ask off, else the profile's user.js (which wins at the start) or prefs.js +// turns it off. Betterbird's own default is to ask. +func (m *Machine) defaultClientAsk(profile string) string { + raw, err := readBounded(m.path(policyFile)) + if err == nil && !strings.Contains(policyProblem(raw), defaultClientPref) { + return "" + } + if profile != "" { + for _, f := range []string{"user.js", "prefs.js"} { + b, err := readBounded(filepath.Join(profile, f)) + if err != nil { + continue + } + if v, ok := readPrefs(b)[defaultClientPref]; ok { + if v == "false" { + return "" + } + return "the profile's " + f + " sets " + defaultClientPref + " to " + v + " and no policy locks it off" + } + } + } + return "no policy locks " + defaultClientPref + " off and Betterbird's default is to ask" +} + // xpiVersion is the version an add-on's manifest.json says, or "" when it cannot be read. func xpiVersion(path string) string { r, err := zip.OpenReader(path) diff --git a/modules/betterbird/cmd/betterbird-tools/install_test.go b/modules/betterbird/cmd/betterbird-tools/install_test.go index 94838b04..ffbb2884 100644 --- a/modules/betterbird/cmd/betterbird-tools/install_test.go +++ b/modules/betterbird/cmd/betterbird-tools/install_test.go @@ -172,3 +172,60 @@ func TestTheUserJSDescriptionSaysAdoption(t *testing.T) { } } } + +// Betterbird never asks at its start to be the default mail client: the module's policy locks +// mail.shell.checkDefaultClient off, and the check says so when it is on (novox/hq issue 345). +func TestTheDefaultClientAskIsLockedOffAndTheCheckSaysWhenItIsOn(t *testing.T) { + f := newFake(t) + f.betterbird() + f.placedByTheMesh() + links(t, nil) + asks := func() string { + t.Helper() + c, err := f.Check() + if err != nil { + t.Fatal(err) + } + for _, x := range c.Findings { + if strings.Contains(x.What, "asks at its start") { + return x.What + " | " + x.Do + } + } + return "" + } + if got := asks(); got != "" { + t.Fatalf("the module's policy locks the ask off, yet: %s", got) + } + unlocked := `{"policies": {"DisableAppUpdate": true, "ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed", + "install_url": "file:///opt/betterbird/owl/owl.xpi", "updates_disabled": true}}}}` + if why := policyProblem([]byte(unlocked)); !strings.Contains(why, defaultClientPref) { + t.Fatalf("a policy that leaves the ask on is said: %q", why) + } + f.write(policyFile, unlocked) + if got := asks(); !strings.Contains(got, "default mail client") || !strings.Contains(got, "push the module") { + t.Fatalf("the ask on is a finding with its remedy: %q", got) + } + f.write(policyFile, `{"policies": {"DisableAppUpdate": true, "Preferences": {"mail.shell.checkDefaultClient": {"Value": false, "Status": "default"}}, + "ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed", "install_url": "file:///opt/betterbird/owl/owl.xpi", "updates_disabled": true}}}}`) + if why := policyProblem(mustRead(t, f, policyFile)); !strings.Contains(why, "locked") { + t.Fatalf("a default the profile can override is not a lock: %q", why) + } + f.write(profileDir+"/prefs.js", `user_pref("mail.shell.checkDefaultClient", true);`+"\n") + if got := asks(); got == "" { + t.Fatal("a profile that turned the ask back on over an unlocked default is a finding") + } + f.write(policyFile, unlocked) + f.write(profileDir+"/prefs.js", `user_pref("mail.shell.checkDefaultClient", false);`+"\n") + if got := asks(); got != "" { + t.Fatalf("a profile with the ask off does not ask: %s", got) + } +} + +func mustRead(t *testing.T, f *fake, p string) []byte { + t.Helper() + raw, err := os.ReadFile(filepath.Join(f.Root, p)) + if err != nil { + t.Fatal(err) + } + return raw +} diff --git a/modules/betterbird/install/distribution/policies.json b/modules/betterbird/install/distribution/policies.json index b2b65390..e75b4c84 100644 --- a/modules/betterbird/install/distribution/policies.json +++ b/modules/betterbird/install/distribution/policies.json @@ -7,6 +7,12 @@ "install_url": "file:///opt/betterbird/owl/owl.xpi", "updates_disabled": true } + }, + "Preferences": { + "mail.shell.checkDefaultClient": { + "Value": false, + "Status": "locked" + } } } } diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index eb3b8c93..a2401df9 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -49,6 +49,23 @@ export interface GiteaIssue { body?: string; } +/** + * **The branch a pull request was opened from, also once it is deleted** (novox/hq issue 578). The forge says it as + * `head.ref` while the branch exists; once it is gone — the merge tool deletes it after the merge by default — + * `head.ref` reads `refs/pull//head`, and only `head.label` still names the branch. A delivery group is the pull + * requests sharing a head branch name (novox/hq ADR 0249), and the controller finds a merge's group, and joins the + * group's merges in one batch, by the head its `pull.merged` names (ADR 0276 decision 5, ADR 0306 decision 2). With + * the pull ref it found no group: on 2026-10-12 mesh-catalog #251's walk was cut at once and went before + * mesh-controller #271's, which the group's order put first. + */ +export function headBranchOf(head: { ref?: unknown; label?: unknown } | null | undefined): string | undefined { + const said = (v: unknown): string | undefined => (typeof v === "string" && v !== "" ? v : undefined); + const ref = said(head?.ref); + const label = said(head?.label); + if (ref === undefined || /^refs\/pull\/\d+\/head$/.test(ref)) return label ?? ref; + return ref; +} + /** A pull request, trimmed to the fields a reviewer or an event body needs. */ export interface GiteaPull { number: number; @@ -341,6 +358,57 @@ export class GiteaClient { return out; } + /** **Every open pull request of a repository**, page by page (novox/hq ADR 0323 §4): what a trunk's move puts out of + * date, however many are open, whatever their update time (a pass announces from listOpenPullsSince). Bounded at + * 20 pages of 50; a repository with more is said, and the rest are left to the merge tool's own comparison. */ + async listOpenPulls(owner: string, repo: string): Promise { + const out: GiteaPull[] = []; + for (let page = 1; page <= 20; page++) { + const batch = await this.listPullRequests(owner, repo, { state: "open", limit: "50", page: String(page) }); + out.push(...batch); + if (batch.length < 50) return out; + } + console.error(`[gitea] ${owner}/${repo} has more than 1000 open pull requests; only the first 1000 are read`); + return out; + } + + /** **The open pull requests of a repository updated since a moment**, newest first, page by page (novox/hq issue + * 579): what a pass announces new heads from. The forge sorts `recentupdate` by update time, newest first, so the + * paging stops at the first page that reaches back past `since`; with no moment — a sweep or the first pass — + * every page is read. Bounded like listOpenPulls at 20 pages of 50. A pull request seen twice, pushed up a page + * while the pages were read, is answered once. */ + async listOpenPullsSince(owner: string, repo: string, since: string): Promise { + const out: GiteaPull[] = []; + const seen = new Set(); + const floor = since ? Date.parse(since) : NaN; + for (let page = 1; page <= 20; page++) { + const batch = await this.listPullRequests(owner, repo, { state: "open", sort: "recentupdate", limit: "50", page: String(page) }); + for (const p of batch) { + if (seen.has(p.number)) continue; + seen.add(p.number); + out.push(p); + } + if (batch.length < 50) return out; + const oldest = batch[batch.length - 1]?.updated_at; + if (!Number.isNaN(floor) && oldest && Date.parse(oldest) < floor) return out; + } + console.error(`[gitea] ${owner}/${repo} has more than 1000 open pull requests updated since ${since || "ever"}; only the newest 1000 are read`); + return out; + } + + /** The commit a branch is at now: a trunk's head (novox/hq ADR 0323). */ + async branchHead(owner: string, repo: string, branch: string): Promise { + const b = await this.request(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`); + const sha = String(b?.commit?.id ?? ""); + if (!/^[0-9a-f]{40,64}$/i.test(sha)) throw new Error(`the forge named no commit for ${owner}/${repo}'s ${branch}`); + return sha; + } + + /** One repository, as the forge holds it: its clone URL above all, for an announcement. */ + async getRepo(owner: string, repo: string): Promise { + return GiteaClient.mapRepo(await this.request(repoPath(owner, repo))); + } + async getPullRequest(owner: string, repo: string, index: number): Promise { indexOf(index); return GiteaClient.mapPull(await this.request(`${repoPath(owner, repo)}/pulls/${index}`)); @@ -646,7 +714,7 @@ export class GiteaClient { merge_commit_sha: p.merge_commit_sha ?? undefined, merged_at: p.merged_at ?? undefined, user: p.user?.login, - head: p.head?.ref, + head: headBranchOf(p.head), head_sha: p.head?.sha ?? undefined, base: p.base?.ref, updated_at: p.updated_at ?? undefined, diff --git a/modules/gitea/gate.ts b/modules/gitea/gate.ts index 9a19ac72..f92510bf 100644 --- a/modules/gitea/gate.ts +++ b/modules/gitea/gate.ts @@ -18,7 +18,7 @@ // restart, when the check was opened is read from the forge's own history of the gate. import { CHECK_CONTEXT, pendingWords } from "./pulls.js"; -import type { CommitStatus } from "./client.js"; +import type { CommitStatus, HeadCheck } from "./client.js"; /** The build seat's `started`, as mesh-controller's link.BuildStart publishes it. */ export interface BuildStarted { @@ -37,6 +37,8 @@ export interface GateForge { gateOf(owner: string, repo: string, sha: string): Promise<{ state: string; updated_at?: string } | undefined>; /** The head's mesh/merge-gate statuses, every one the forge keeps, newest first (novox/hq issue 456). */ gateHistory(owner: string, repo: string, sha: string): Promise; + /** The head's statuses by context, the newest of each, with their words (novox/hq ADR 0323). */ + headChecks(owner: string, repo: string, sha: string): Promise>; } /** One status of a head's gate as the forge keeps it: its state, its words, and when it was set. */ @@ -174,6 +176,26 @@ export class GateWriter { }); } + /** + * **A trunk's move puts the head's verdict out of date** (novox/hq ADR 0323 §4): its merge check statuses are put back + * to pending, with the reason, in their turn after every write of the head already asked for — a verdict in flight + * lands first. What is written is decided in the turn, from the statuses the forge holds then (`decide`, trunk.ts's + * outOfDateStatuses): a check on its way, or a verdict already judged against the trunk's head, is left as it is. It + * opens no check — nothing is asked — so a verdict of a check asked before it keeps its weight. Answers what it wrote; + * a write that fails fails the caller, and the move is weighed again on the next pass. + */ + outOfDate(owner: string, repo: string, sha: string, + decide: (checks: Record) => CommitStatus[] | Promise, target?: string): Promise { + const h = this.head(owner, repo, sha); + const written: CommitStatus[] = []; + return this.turn(h, async () => { + for (const s of await decide(await this.forge.headChecks(owner, repo, sha))) { + await this.forge.setCommitStatus(owner, repo, sha, target ? { ...s, target_url: target } : s); + written.push(s); + } + }).then(() => written); + } + /** * A build machine took an ask of this head: said while no verdict was heard, and only for the check now pending — * a start from before the pending words were set (a run before a recheck, the stream's history) is not this check's. diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 4cbd90fa..d64045a7 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -17,8 +17,13 @@ // Consumes node-build-agent.started — a build machine took an ask — and says, on the head a pending check is of, // which machine builds it since when (novox/hq issue 446). This module is the one writer of mesh/merge-gate. // -// Serves gitea_commit_status from this bundle, not the tools', so a recheck's pending reaches that one writer -// (novox/hq issue 456). +// **A trunk's move puts every open verdict into it out of date** (novox/hq ADR 0323 §4, trunk.ts): each pass reads the +// head of every trunk an open pull request merges into, and when one moved, the merge check statuses of every open pull +// request into it go back to pending with the reason. Nothing is run for it. A verdict that lands judged against a trunk +// commit that is no longer the trunk's head is said out of date in place of its pass or failure. +// +// Serves gitea_commit_status and gitea_merge_pull_request from this bundle, not the tools', so a recheck's pending and +// the merge tool's ask of a check reach that one writer (novox/hq issue 456, ADR 0323). // // Every pull request the forge holds is announced, whatever its repository: the controller holds the // module graph and decides what is checked — a repository is never asked to opt in. @@ -34,10 +39,12 @@ import { emit, on } from "@novox/mesh-sdk/events"; import { registerModuleTools } from "@novox/mesh-sdk/tools"; import { commitStatusTool } from "./delivery.js"; +import { mergeTool } from "./merge.js"; import { GiteaClient, movedSince } from "./client.js"; import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, type RepoCheckFacts, reposToLook } from "./pulls.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; +import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is // nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints @@ -245,42 +252,15 @@ async function pollPulls(client: GiteaClient): Promise { } if (sweep) lastSweep = Date.now(); for (const repo of reposToLook(await client.listAllRepos(), floor, updated, sweep)) { - const open = await client.listPullRequests(repo.owner, repo.name, { state: "open", sort: "recentupdate", limit: "20" }); + // Every open pull request updated since the floor, paged (novox/hq issue 579): the newest 20 alone left a head + // beyond them unannounced. A sweep and the first pass read every page. + const open = await client.listOpenPullsSince(repo.owner, repo.name, sweep ? "" : floor); for (const pull of headsToAnnounce(repo.full_name, open, heads)) { const key = `${repo.full_name}#${pull.number}`; const fresh = primedPulls || (!!pull.updated_at && pull.updated_at > dayAgo); if (fresh) { - const files = await client.listPullFiles(repo.owner, repo.name, pull.number); - const head = String(pull.head_sha); - // What the controller maps the change onto the mesh's module graph with (novox/hq ADR 0237 as - // amended): which changed directories hold a module at the head — the merge's rule (issue 278) — - // and whether the head holds the repository's own merge-check.sh. Not said when it could not be - // read; the controller then reads the change as touching everything built from the repository. - let moduleDirs: string[] | null = null; - let mergeCheck: boolean | null = null; - try { - if (!files.truncated) moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, head, files.paths); - mergeCheck = await client.holdsFile(repo.owner, repo.name, head, "merge-check.sh"); - } catch (err) { - console.error(`[gitea] ${key}: what the head holds could not be read — ${err instanceof Error ? err.message : err}`); - } - await emit("pull.updated", { - owner: repo.owner, - repo: repo.name, - number: pull.number, - title: pull.title, - body: pull.body, - base: pull.base, - head: pull.head, - head_sha: pull.head_sha, - clone_url: repo.clone_url, - html_url: pull.html_url, - paths: files.paths, - paths_truncated: files.truncated, - removed: files.removed, - ...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}), - ...(mergeCheck !== null ? { merge_check: mergeCheck, merge_check_said: true } : {}), - }); + // The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5). + await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url, pull)); // Said, so an operator knows the mesh was asked to check it. console.log(`[gitea] announced ${key} at ${String(pull.head_sha).slice(0, 8)} to be checked before it merges`); // Pending until the verdict comes, so the pull request says a check is running rather than nothing — and @@ -290,17 +270,52 @@ async function pollPulls(client: GiteaClient): Promise { heads[key] = String(pull.head_sha); changed = true; } + if (await trunksMoved(client, repo.owner, repo.name)) changedTrunks = true; } if (!primedPulls || changed) keepHeads(); + if (changedTrunks) keepTrunks(); + changedTrunks = false; primedPulls = true; lastPullLook = began; } +// **The trunks' heads, as this module last read them** (novox/hq ADR 0323 §4), by `owner/repo:branch`: kept beside the +// other records, so a move made while this module was away is weighed on its first pass back. With no record — the +// first pass after this rolls out — a trunk's head is taken as it is and nothing is put out of date: the verdicts +// then standing are weighed on the trunk's next move, and the merge tool compares commits whatever this holds. +const trunksRecord = process.env.MESH_GITEA_STATE_DIR ? join(process.env.MESH_GITEA_STATE_DIR, "trunks-seen.json") : null; +let trunks: Record = {}; +let changedTrunks = false; +if (trunksRecord && existsSync(trunksRecord)) { + try { + trunks = (JSON.parse(readFileSync(trunksRecord, "utf8")) as { trunks: Record }).trunks ?? {}; + } catch { + // An unreadable record is no record: the next pass takes each trunk's head as it is. + } +} +function keepTrunks(): void { + if (!trunksRecord) return; + mkdirSync(join(trunksRecord, ".."), { recursive: true }); + const tmp = trunksRecord + ".tmp"; + writeFileSync(tmp, JSON.stringify({ trunks })); + renameSync(tmp, trunksRecord); +} + +/** Whether a trunk of one repository moved, weighed with this module's forge and its one writer (trunk.ts). */ +function trunksMoved(client: GiteaClient, owner: string, name: string): Promise { + return weighTrunks(client, gates, trunks, owner, name); +} + // **The verdict, set where the pull request shows it.** Every verdict is the head commit's status; one // that is not a pass also leaves the check's own account as a comment, so the reason is read where the // change is reviewed. An error — the check could not run — is the forge's `error`, never a success. async function setVerdict(client: GiteaClient, event: { body: unknown }): Promise { const c = (event.body ?? {}) as Checked; + if (!c.group && c.owner && c.repo && c.took) { + // How long it took, for the merge tool to say how long the next one may (novox/hq ADR 0323 §5). + keepTook(tookFile(process.env.MESH_GITEA_STATE_DIR), `${c.owner}/${c.repo}`, + { took: c.took, at: new Date().toISOString(), head: c.commit }); + } if (c.group) { // A delivery group's composed check (novox/hq ADR 0239): its verdict is the group owner's to say, on each // member's head, as mesh/delivery-group — never this head's merge gate. @@ -318,6 +333,26 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis target = pull?.html_url || undefined; base = pull?.base || undefined; } + // **A verdict judged against a trunk commit that is no longer the trunk's head is out of date when it lands** + // (novox/hq ADR 0323 §4): a check asked before the trunk moved. It is said so, pending with the reason, in place of + // its pass or failure; a trunk that cannot be read lets the verdict stand, and the merge tool compares the commits. + const trunk = base ?? c.plan?.base; + if (c.trunk && trunk) { + const head = await client.branchHead(c.owner, c.repo, trunk).catch(() => undefined); + if (head && !sameCommit(c.trunk, head)) { + const description = outOfDateWords(c.trunk, head); + for (const context of [CHECK_CONTEXT, ...(c["repo-check"] ? [REPO_CHECK_CONTEXT] : [])]) { + await client.setCommitStatus(c.owner, c.repo, c.commit, { state: "pending", context, description, + ...(target ? { target_url: target } : {}) }); + } + // Its account is kept and shown all the same (ADR 0323 §7): what it judged, and that it is out of date. + const said = commentFor(c, await repoCheckFacts(client, c, trunk)); + if (c.number) await client.addComment(c.owner, c.repo, c.number, lateVerdictComment(c, said, description)); + console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}, ` + + `${description}`); + return; + } + } const facts = await repoCheckFacts(client, c, base ?? c.plan?.base); for (const status of statusesFor(c, facts)) { await client.setCommitStatus(c.owner, c.repo, c.commit, target ? { ...status, target_url: target } : status); @@ -350,6 +385,7 @@ const gates = new GateWriter({ setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), gateOf: async (owner, repo, sha) => (await (gitea as GiteaClient).headChecks(owner, repo, sha))[CHECK_CONTEXT], gateHistory: (owner, repo, sha) => (gitea as GiteaClient).contextHistory(owner, repo, sha, CHECK_CONTEXT), + headChecks: (owner, repo, sha) => (gitea as GiteaClient).headChecks(owner, repo, sha), }); // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own @@ -358,7 +394,7 @@ const gates = new GateWriter({ registerModuleTools("gitea", () => (gitea ? [commitStatusTool({ setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), gates, -})] : [])); +}), mergeTool(gitea, { gates })] : [])); // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing. diff --git a/modules/gitea/merge.ts b/modules/gitea/merge.ts new file mode 100644 index 00000000..05ee77a2 --- /dev/null +++ b/modules/gitea/merge.ts @@ -0,0 +1,183 @@ +// The forge's merge tool, `gitea_merge_pull_request` (novox/hq ADR 0266, issue 381, ADR 0323). +// +// **Served from the watcher's bundle, beside the merge check's one writer** (index.ts, gate.ts; novox/hq issue 456): +// when a verdict is judged against an earlier trunk commit than the trunk's head, the tool asks the head's check and +// puts its statuses back to pending through that writer, so where the check runs is said and no second check is asked. +// Each bundle the runtime serves is its own process; from the tools' bundle the pending bypassed the writer's chain. + +import { emit } from "@novox/mesh-sdk/events"; +import type { ToolDefinition } from "@novox/mesh-sdk/tools"; +import type { CommitStatus, GiteaClient } from "./client.js"; +import { mergeableUnder, newestByContext } from "./checks.js"; +import { MERGE_METHODS, Refused, checkName } from "./guard.js"; +import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js"; +import { announcementOf, askedWords, outOfDateWords, tookFile, tookWords, twelve, verdictAtTrunk } from "./trunk.js"; + +/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */ +const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/; + +/** What the merge tool reaches beside the forge: the merge check's one writer, how an event is said, the module's + * state directory, and the clock. */ +export interface MergeDeps { + gates: { rechecked(owner: string, repo: string, sha: string, status: CommitStatus): Promise }; + emit?: (type: string, body: unknown) => Promise; + stateDir?: string; + now?: () => Date; +} + +export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition { + const deps = { ...given, emit: given.emit ?? emit, stateDir: given.stateDir ?? process.env.MESH_GITEA_STATE_DIR, + now: given.now ?? (() => new Date()) }; + return { + name: "gitea_merge_pull_request", + description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head " + + "and every status its base branch's protection requires succeeded there (mesh/merge-gate, mesh/repo-check), " + + "judged against the trunk's head as it is now, optionally deleting the source branch afterwards. Refused " + + "otherwise, with nothing merged: no tool merges past a check, and none merges a head that moved after it was " + + "reviewed. A verdict judged against an earlier trunk commit is refused, naming both, and the head's check against " + + "the trunk's head is asked once, with how long the last check of the repository took: ask again at the same head " + + "once it passes — the tool never merges by itself.", + input: { + owner: { type: "string", description: "the repository owner" }, + repo: { type: "string", description: "the repository name" }, + number: { type: "number", description: "the PR number" }, + head: { type: "string", description: "the commit you reviewed, as its sha or at least its first 12 characters: " + + "refused unless it is the pull request's head" }, + method: { type: "string", description: "merge (default) | fast-forward-only | squash | rebase | rebase-merge. " + + "Only merge and fast-forward-only can be read back to prove the head you named is what landed" }, + delete_branch: { type: "boolean", description: "delete the source branch after merge (default true)" }, + }, + run: async (args) => { + const owner = checkName("owner", args.owner); + const repo = checkName("name", args.repo); + const number = Number(args.number); + const method = args.method ? String(args.method) : "merge"; + if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`); + const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch); + // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left + // to the forge, whose administrator — the account this tool's token is minted for — may merge past a + // rule that lets it. A rule that cannot be read is no rule passed. + // **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d, + // force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the + // head was and nothing told it which head the review had approved. The caller names it; a head that moved + // since is refused, and the forge is handed the same head, so a push between this read and the merge is + // refused by the forge too. + const named = args.head === undefined ? "" : String(args.head).trim().toLowerCase(); + if (!named) { + throw new Refused(`${owner}/${repo}#${number} is not merged: name the head you reviewed (head), so that only ` + + `that commit can land (novox/hq issue 381). Nothing was merged`); + } + // Twelve characters, not seven: an author who force-pushes can make a commit whose first seven match the + // reviewed one in seconds, and the review would then merge it. Twelve cannot be matched that way. + if (!REVIEWED_HEAD.test(named)) { + throw new Refused(`${owner}/${repo}#${number} is not merged: head ${named} does not name one commit — give its ` + + `sha, or at least its first 12 characters. Nothing was merged`); + } + const pull = await gitea.getPullRequest(owner, repo, number); + if (pull.merged) throw new Refused(`${owner}/${repo}#${number} is already merged`); + if (!pull.head_sha || !pull.base) throw new Refused(`${owner}/${repo}#${number} names no head commit or base branch`); + if (!pull.head_sha.toLowerCase().startsWith(named)) { + throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` + + `${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`); + } + const rule = await gitea.branchProtection(owner, repo, pull.base); + // Only into a branch whose protection requires the checks: a pull request whose base was moved to a + // branch nothing guards would merge past every check (novox/hq ADR 0266). + if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) { + throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule requiring ` + + `the checks guards: a tool merges only into such a branch (novox/hq ADR 0266). Nothing was merged`); + } + // **The verdict must be judged against the trunk's head** (novox/hq ADR 0323 §5, issue 341): a check judges the + // head merged into a named trunk commit, and a merge of another trunk is not the one that check judged. Compared + // here at the moment the merge is asked, not left to the statuses the trunk's move resets a pass later. When they + // differ nothing is merged, and the head's check against the trunk's head is asked once — the announcement a new + // head makes — never again while one is on its way. The caller asks again at the reviewed head once it passes. + const trunkHead = await gitea.branchHead(owner, repo, pull.base); + const checks = await gitea.headChecks(owner, repo, pull.head_sha); + const at = verdictAtTrunk(checks, trunkHead); + if (!at.current) { + const judged = at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against"; + const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`); + const head = twelve(pull.head_sha), now = twelve(trunkHead); + if (at.onItsWay) { + throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` + + `${now}. Its check is already asked and not yet answered (${checks[CHECK_CONTEXT]?.description}), so it is not ` + + `asked again; ${took}. Ask again at ${head} once both statuses pass: this tool never merges by itself ` + + `(novox/hq ADR 0323). A check that never answers is asked again with mesh-delivery.recheck. Nothing was merged`); + } + // Pending first, so a second ask finds the check on its way; then the head is announced as a new head is. An + // announcement that cannot be made puts the statuses back to out of date, so the next ask asks again rather + // than wait for ever on a check nobody asked. + const contexts = [CHECK_CONTEXT, ...(checks[REPO_CHECK_CONTEXT] ? [REPO_CHECK_CONTEXT] : [])]; + // Through the module's one writer of the merge check (gate.ts, novox/hq issue 456): the pending opens a new + // check of the head, so where it runs is said ("building on …") and its verdict is not weighed as older. + const say = async (description: string) => { + for (const context of contexts) { + await deps.gates.rechecked(owner, repo, String(pull.head_sha), { state: "pending", context, description, + ...(pull.html_url ? { target_url: pull.html_url } : {}) }); + } + }; + await say(askedWords(deps.now(), trunkHead)); + try { + const cloneURL = (await gitea.getRepo(owner, repo)).clone_url; + await deps.emit("pull.updated", await announcementOf(gitea, owner, repo, cloneURL, pull)); + } catch (err) { + await say(outOfDateWords(at.judged, trunkHead)).catch(() => {}); + throw new Error(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` + + `${now}, and its check could not be asked: ${err instanceof Error ? err.message : err}. Nothing was merged`); + } + throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` + + `${now}. The check of head ${head} merged into trunk ${now} was asked; ${took}. Ask again at ${head} once both ` + + `statuses pass: this tool never merges by itself (novox/hq ADR 0323). In a busy repository another merge may ` + + `move the trunk again first; this is then refused and asked once more. A check that never answers is asked ` + + `again with mesh-delivery.recheck. Nothing was merged`); + } + const statuses = newestByContext((await gitea.combinedStatus(owner, repo, pull.head_sha)).statuses); + const verdict = mergeableUnder(pull.base, rule, statuses); + if (verdict.mergeable !== true) { + throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` + + `its checks succeed; no tool merges past them (novox/hq ADR 0266)`); + } + await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha); + // The merge commit only exists now; answered so the caller can follow what is built from it. + // pull.merged is the events entrypoint's to announce (index.ts), once, within its poll. + // From here the merge has happened. Nothing below may read as if it had not: a read-back that fails or cannot + // be done is said beside merged: true, and only a proven other head is an error — one that says the merge happened. + const answer: Record = { merged: true, number, method, head: pull.head_sha, deleted_branch: deleteBranch }; + let landed: string | undefined; + try { + const merged = await gitea.getPullRequest(owner, repo, number); + answer.merge_commit_sha = merged.merge_commit_sha; + // What landed, read back from the commit the merge made rather than from the forge's record of the pull + // request, which is the very thing a race would leave behind. A merge commit's second parent is the head it + // joined; a fast-forward lands the head itself. A squash or a rebase makes new commits that name no head. + if (!merged.merge_commit_sha) { + answer.read_back = "the forge named no merge commit, so what landed cannot be read back"; + } else if (method === "merge") { + const parents = await gitea.commitParents(owner, repo, merged.merge_commit_sha); + if (parents.length === 2) landed = parents[1]; + else answer.read_back = `the merge commit has ${parents.length} parent(s), not two: what landed cannot be read back`; + } else if (method === "fast-forward-only") { + // That the forge names the head itself as a fast-forward's merge commit is assumed, not seen on this forge: + // a different sha is said, not raised as another head having landed. + if (merged.merge_commit_sha === pull.head_sha) landed = merged.merge_commit_sha; + else answer.read_back = `the forge named ${merged.merge_commit_sha.slice(0, 12)} as this fast-forward's commit, ` + + `not the head: that the head landed could not be confirmed`; + } else { + answer.read_back = `a ${method} makes new commits that name no head: what landed cannot be read back`; + } + } catch (err) { + answer.read_back = `the merge happened, but reading back what landed failed: ${(err as Error).message}`; + } + // Loud, not a field beside merged: true that a caller reading only that would miss. The merge cannot be taken + // back by this tool; what must not happen is that it reads as the merge that was asked for. + if (landed !== undefined && landed !== pull.head_sha) { + throw new Error(`${owner}/${repo}#${number} WAS merged (${answer.merge_commit_sha}), but the forge merged ` + + `${landed.slice(0, 12)}, not the ${pull.head_sha.slice(0, 12)} it was handed and you reviewed: a commit nobody ` + + `reviewed has landed. Read it before anything is built from it (novox/hq issue 381)`); + } + if (landed !== undefined) answer.landed = landed; + return answer; + }, + }; +} diff --git a/modules/gitea/pulls.ts b/modules/gitea/pulls.ts index 4e0ec815..a51156a1 100644 --- a/modules/gitea/pulls.ts +++ b/modules/gitea/pulls.ts @@ -59,6 +59,12 @@ export interface Checked { plan?: ChangePlan; /** Set on a delivery group's composed check (novox/hq ADR 0239): not this head's merge gate. */ group?: string; + /** The trunk commit the head was merged into and judged as, twelve characters (novox/hq ADR 0323 §3); absent from a + * controller whose build seat judged the head alone. Beside are the other repositories' commits the check read, + * by directory; took how long it ran. */ + trunk?: string; + beside?: Record; + took?: string; } /** A change plan, as the controller says it (mesh-controller internal/link, ChangePlan). */ @@ -199,17 +205,25 @@ export function statusFor(c: Checked): CommitStatus { const description = builds(c.plan) ? described(gate.verdict, `${c.plan!.summary}; ${gate.summary}`) : described(gate.verdict, gate.summary, gate.modules, gate.dependents); - return { state: stateOf(gate.verdict), context: CHECK_CONTEXT, description }; + return judgedFirst(c, { state: stateOf(gate.verdict), context: CHECK_CONTEXT, description }); } /** Every status a verdict sets: the gate's, and the repository's own check's when it was said. */ export function statusesFor(c: Checked, facts: RepoCheckFacts = {}): CommitStatus[] { const out = [statusFor(c)]; const repo = c["repo-check"]; - if (repo) out.push(repoStatus(repo, facts)); + if (repo) out.push(judgedFirst(c, repoStatus(repo, facts))); return out; } +/** **A status names what its verdict judged, first** (novox/hq ADR 0323 §3): "head 1a2b3c4d5e6f merged into trunk + * 7a8b9c0d1e2f — pass: …", so the forge's 140 characters never clip it away; the trunk's move is weighed by it. A + * verdict that names no trunk commit is said as before. */ +function judgedFirst(c: Checked, status: CommitStatus): CommitStatus { + if (!c.trunk) return status; + return { ...status, description: clipped(`head ${c.commit.slice(0, 12)} merged into trunk ${c.trunk.slice(0, 12)} — ${status.description}`) }; +} + /** The comment a verdict leaves on its pull request, with the check's own account: the change plan of a * change that builds something (novox/hq ADR 0238), and why, when the gate is not a pass or the * repository's own check failed or could not run. A repository with no merge-check.sh, touching nothing, @@ -220,6 +234,12 @@ export function commentFor(c: Checked, facts: RepoCheckFacts = {}): string | nul const wrong = repoWrong(repo, facts); if (gate.verdict === "pass" && !wrong && !builds(c.plan)) return null; const lines = [`**Merge check** at \`${c.commit.slice(0, 8)}\``, ""]; + if (c.trunk) { + // What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and what was read beside it. + const beside = Object.entries(c.beside ?? {}).map(([d, sha]) => `${d} \`${sha}\``).join(", "); + lines.splice(1, 0, `Judged: head \`${c.commit.slice(0, 12)}\` merged into trunk \`${c.trunk.slice(0, 12)}\`` + + (beside ? `; read beside it: ${beside}` : "")); + } lines.push(`- \`${CHECK_CONTEXT}\`: **${saidAs(gate.verdict).toUpperCase()}** — ${gate.summary}` + (gate.modules?.length ? ` (modules: ${gate.modules.join(", ")}` + (gate.dependents?.length ? `; built after them: ${gate.dependents.join(", ")}` : "") + ")" : "")); diff --git a/modules/gitea/test/announcing.test.ts b/modules/gitea/test/announcing.test.ts index 19ab6f8a..b24c7fef 100644 --- a/modules/gitea/test/announcing.test.ts +++ b/modules/gitea/test/announcing.test.ts @@ -93,3 +93,47 @@ test("a pull request or issue number that is not one is refused before the forge server.close(); } }); + +// **A merge names the branch it came from, also once that branch is deleted** (novox/hq issue 578). On 2026-10-12 +// the merge tool merged mesh-controller #271 and mesh-catalog #251, one delivery group, and deleted each branch. The +// forge then read each pull request's head.ref as `refs/pull//head`, and `pull.merged` named that as the head: the +// controller asked mesh-delivery for a group of that name, found none, and cut #251's walk at once, before #271's, +// which the group's order put first. +test("a merged pull request whose branch was deleted is announced with its branch, not its pull ref", async () => { + const { GiteaClient, headBranchOf } = await import("../client.ts"); + const group = "feat/341-a-merge-check-judges-the-merge"; + // As the forge answered for #251 after its merge. + const merged = { + number: 251, title: "A verdict goes out of date when its trunk moves", state: "closed", merged: true, + merged_at: "2026-10-12T16:11:29Z", merge_commit_sha: "23ea0f431faf91aa83e7b0297e0a9e442180f0c7", + head: { label: group, ref: "refs/pull/251/head", sha: "41b6e10077efdef694b519879fcbaeaf69d472b1" }, + base: { label: "main", ref: "main" }, html_url: "http://fake/novox/mesh-catalog/pulls/251", + }; + const server = createServer((req, res) => { + const url = req.url ?? ""; + const send = (v: unknown) => res.end(JSON.stringify(v)); + if (url.startsWith("/api/v1/repos/novox/mesh-catalog/pulls/251")) return send(merged); + if (url.startsWith("/api/v1/repos/novox/mesh-catalog/pulls?")) return send([merged]); + res.statusCode = 404; + res.end("{}"); + }); + await new Promise((r) => server.listen(0, r)); + const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t"); + try { + // What the announcer reads each pass, and what `pull.merged` carries as its head. + const listed = await client.listPullRequests("novox", "mesh-catalog", { state: "closed", sort: "recentupdate", limit: "20" }); + assert.deepEqual(listed.map((p) => p.head), [group], "the pass names the group's branch, never refs/pull/251/head"); + assert.equal((await client.getPullRequest("novox", "mesh-catalog", 251)).head, group); + } finally { + server.close(); + } + // While the branch exists the forge says it as ref, and that is what is named; a label alone is the branch too. + assert.equal(headBranchOf({ label: "feat/x", ref: "feat/x" }), "feat/x"); + assert.equal(headBranchOf({ label: "feat/x" }), "feat/x"); + assert.equal(headBranchOf({ ref: "feat/x" }), "feat/x"); + // A branch whose own name only looks like a pull ref's start is still a branch. + assert.equal(headBranchOf({ label: "y", ref: "refs/pulled" }), "refs/pulled"); + // No label to read: the pull ref is all the forge said, and it is kept rather than made up. + assert.equal(headBranchOf({ ref: "refs/pull/7/head" }), "refs/pull/7/head"); + assert.equal(headBranchOf(undefined), undefined); +}); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 3e069b63..f14b50ec 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -7,11 +7,14 @@ import { test, after, beforeEach } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { GiteaClient } from "../dist/client.js"; import { ConfiguredToken } from "../dist/token.js"; import { getGiteaTools } from "../dist/tools/index.js"; +import { mergeTool } from "../dist/merge.js"; import { commitStatusTool } from "../dist/delivery.js"; import { GateWriter } from "../dist/gate.js"; import { judgeApiCall, judgeProtectionChange, ruleCovers, routedPath } from "../dist/guard.js"; @@ -21,7 +24,11 @@ interface Sent { method: string; path: string; body: any } const state = { sent: [] as Sent[], asked: [] as string[], - statuses: [] as { context: string; state: string }[], + statuses: [] as { context: string; state: string; description?: string; updated_at?: string }[], + // The trunk's head (novox/hq ADR 0323): what the merge tool compares the verdict's trunk commit with. + trunk: "", + // What the tools said on the bus. + emitted: [] as { type: string; body: any }[], rules: [] as any[], lastCommit: "2026-01-01T00:00:00Z", // The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's @@ -36,13 +43,21 @@ const state = { commitsFail: 0, }; const REVIEWED = "abc1234def5678901234567890abcdef12345678"; +const TRUNK = "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"; +const MOVED = "3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d"; +// A verdict judged against a trunk commit, as both statuses say it (novox/hq ADR 0323 §3). +const judgedAt = (trunk: string, state = "success") => [ + { context: "mesh/merge-gate", state, description: `head ${REVIEWED.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: x` }, + { context: "mesh/repo-check", state, description: `head ${REVIEWED.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: y` }]; const protectedMain = () => ({ rule_name: "main", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], block_admin_merge_override: true }); beforeEach(() => { state.sent = []; state.asked = []; - state.statuses = [{ context: "mesh/merge-gate", state: "success" }, { context: "mesh/repo-check", state: "success" }]; + state.statuses = judgedAt(TRUNK); + state.trunk = TRUNK; + state.emitted = []; state.rules = [protectedMain()]; state.head = REVIEWED; state.landed = ""; @@ -73,6 +88,15 @@ const server = createServer(async (req, res) => { const repo = "/repos/novox/mesh-catalog"; if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" }); if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules); + if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } }); + if (method === "GET" && path === `${repo}/pulls/7/files`) { + return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []); + } + if (method === "POST" && path === `${repo}/statuses/${state.head}`) { + // Kept as the forge keeps them: the newest of a context first. + state.statuses = [{ ...body, updated_at: new Date().toISOString() }, ...state.statuses.filter((s) => s.context !== body.context)]; + return json(res, 201, body); + } if (method === "GET" && path === "/user") return json(res, 200, { login: "mesh-admin" }); if (method === "GET" && path === "/repos/mesh-admin/scratch/branch_protections") return json(res, 200, []); if (method === "GET" && /^\/repos\/[^/]+\/[^/]+\/commits$/.test(path)) { @@ -102,11 +126,19 @@ after(() => new Promise((r) => server.close(() => r()))); const { port } = server.address() as { port: number }; const client = new GiteaClient(`http://127.0.0.1:${port}`, new ConfiguredToken("t")); // gitea_commit_status is served from the watcher's bundle, beside the merge check's one writer (novox/hq issue 456). +const stateDir = mkdtempSync(join(tmpdir(), "gitea-tools-")); +// The merge check's one writer, as the watcher's bundle holds it: what the merge tool's pending goes through. +const writer = new GateWriter({ setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), + gateOf: async () => undefined, gateHistory: async () => [], headChecks: async () => ({}) }); +const throughTheWriter: string[] = []; +const gates = { rechecked: (o: string, r: string, sha: string, s: any) => { + throughTheWriter.push(`${sha.slice(0, 12)} ${s.context} ${s.state}`); + return writer.rechecked(o, r, sha, s); +} }; const tools = [...getGiteaTools(client), commitStatusTool({ - setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), - gates: new GateWriter({ setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), - gateOf: async () => undefined, gateHistory: async () => [] }), -})]; + setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), gates: writer, +}), mergeTool(client, { gates, stateDir, emit: async (type, body) => { state.emitted.push({ type, body }); }, + now: () => new Date(2026, 9, 12, 15, 1) })]; const tool = (name: string) => tools.find((t) => t.name === name)!; const writes = () => state.sent.map((s) => `${s.method} ${s.path}`); @@ -189,15 +221,15 @@ test("gitea_branch_protection_set only strengthens", async () => { test("gitea_merge_pull_request merges only a head whose required statuses all succeeded, and only that head", async () => { const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }); - state.statuses = [{ context: "mesh/merge-gate", state: "success" }, { context: "mesh/repo-check", state: "failure" }]; + state.statuses = [judgedAt(TRUNK)[0], judgedAt(TRUNK, "failure")[1]]; await assert.rejects(merge(), /not merged: .*mesh\/repo-check \(failure\)/); - state.statuses = [{ context: "mesh/merge-gate", state: "success" }]; + state.statuses = [judgedAt(TRUNK)[0]]; await assert.rejects(merge(), /mesh\/repo-check \(missing\)/); await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, method: "manually-merged" }), /not a merge method/); assert.deepEqual(writes(), []); // The operator said yes and the checks passed: the flow that stays. - state.statuses = [{ context: "mesh/merge-gate", state: "success" }, { context: "mesh/repo-check", state: "success" }]; + state.statuses = judgedAt(TRUNK); await merge(); const sent = state.sent.find((s) => s.path.endsWith("/pulls/7/merge"))!; assert.equal(sent.body.force_merge, false); @@ -205,6 +237,75 @@ test("gitea_merge_pull_request merges only a head whose required statuses all su }); +// **The merge tool refuses a verdict whose trunk commit is not the trunk's head, and asks the check itself** (novox/hq +// ADR 0323 §5, issue 341): node-engine #66 merged at 00:54 on a verdict of 00:49 that judged its head alone, after #65 +// had moved the trunk. Both commits are named; one check is asked, the announcement a new head makes, and not a second +// while it is on its way; the refusal says how long the last check of the repository took; and once the verdict judges +// the trunk's head and passes, the merge goes through — asked again, never by itself. +test("gitea_merge_pull_request refuses a verdict judged against an earlier trunk, and asks the check once", async () => { + const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }); + const { keepTook, tookFile } = await import("../dist/trunk.js"); + keepTook(tookFile(stateDir), "novox/mesh-catalog", { took: "4m31s", at: "2026-10-12T14:49:00Z" }); + + state.trunk = MOVED; // another merge moved the trunk after the verdict + await assert.rejects(merge(), (err: Error) => { + assert.match(err.message, /is not merged: its verdict was judged against trunk 7a8b9c0d1e2f, and main is now 3c4d5e6f7a8b/); + assert.match(err.message, /check of head abc1234def56 merged into trunk 3c4d5e6f7a8b was asked/); + assert.match(err.message, /the last check of novox\/mesh-catalog took 4m31s/); + assert.match(err.message, /never merges by itself/); + return true; + }); + assert.equal(state.merged, false); + assert.equal(state.emitted.length, 1, "one check asked"); + assert.equal(state.emitted[0].type, "pull.updated"); + assert.equal(state.emitted[0].body.head_sha, REVIEWED); + assert.equal(state.emitted[0].body.base, "main"); + assert.deepEqual(state.emitted[0].body.paths, ["modules/gitea/x.ts"]); + // The announcement names the trunk's head, for a verdict no build judges (ADR 0323 §3). + assert.equal(state.emitted[0].body.base_sha, MOVED); + // Its pending went through the merge check's one writer (novox/hq issue 456), not around it. + assert.deepEqual(throughTheWriter.splice(0), ["abc1234def56 mesh/merge-gate pending", "abc1234def56 mesh/repo-check pending"]); + const gate = state.statuses.find((s) => s.context === "mesh/merge-gate")!; + assert.equal(gate.state, "pending"); + assert.match(gate.description!, /^waiting in the build queue since 15:01: asked by the merge tool against trunk 3c4d5e6f7a8b/); + assert.equal(state.statuses.find((s) => s.context === "mesh/repo-check")!.state, "pending"); + + // Asked again while that check is on its way: refused, and nothing is asked a second time. + await assert.rejects(merge(), /already asked and not yet answered .*not asked again; the last check of novox\/mesh-catalog took 4m31s.*mesh-delivery\.recheck/); + assert.equal(state.emitted.length, 1, "no second check while one is on its way"); + state.statuses = [{ ...gate, description: "building on ace since 15:02" }, ...state.statuses.slice(1)]; + await assert.rejects(merge(), /not asked again/); + assert.equal(state.emitted.length, 1); + + // A verdict put out of date by the trunk's move is asked too, once. + state.statuses = judgedAt(TRUNK).map((s) => ({ ...s, state: "pending", + description: "out of date: judged against trunk 7a8b9c0d1e2f, the trunk is now 3c4d5e6f7a8b; judged again on a recheck or when it is merged" })); + await assert.rejects(merge(), /was judged against trunk 7a8b9c0d1e2f, and main is now 3c4d5e6f7a8b\. The check .* was asked/); + assert.equal(state.emitted.length, 2); + + // A verdict that names no trunk commit — a build seat from before the rule — is not taken as the trunk's head. + state.statuses = [{ context: "mesh/merge-gate", state: "success", description: "pass: x" }, + { context: "mesh/repo-check", state: "success", description: "pass: y" }]; + await assert.rejects(merge(), /names no trunk commit it was judged against, and main is now 3c4d5e6f7a8b/); + assert.equal(state.emitted.length, 3); + assert.equal(state.merged, false); + + // An ask that cannot be announced puts the statuses back to out of date, so the next ask asks again. + state.statuses = judgedAt(TRUNK); + const failing = mergeTool(client, { gates, stateDir, emit: async () => { throw new Error("the bus is away"); } }); + await assert.rejects(failing.run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }), + /its check could not be asked: the bus is away\. Nothing was merged/); + assert.match(state.statuses.find((s) => s.context === "mesh/merge-gate")!.description!, /^out of date: /); + await assert.rejects(merge(), /was asked/); + assert.equal(state.emitted.length, 4, "asked again after the announcement failed"); + + // The check judged the merge into the trunk's head and passed: asked again at the reviewed head, it merges. + state.statuses = judgedAt(MOVED); + const done = (await merge()) as Record; + assert.equal(done.merged, true); + assert.equal(state.emitted.length, 4, "a merge asks no check"); +}); + // A merge lands only the head that was reviewed (novox/hq issue 381): mesh-controller#197 was reviewed at one head, // force-pushed, checked green at the new one, and merged there, because the tool merged whatever the head was. test("gitea_merge_pull_request merges only the head it is named, and refuses one that is not the head", async () => { diff --git a/modules/gitea/test/paging.test.ts b/modules/gitea/test/paging.test.ts new file mode 100644 index 00000000..a9f40a32 --- /dev/null +++ b/modules/gitea/test/paging.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createServer } from "node:http"; + +// **Every open pull request's new head is announced, however many are open** (novox/hq issue 579). A pass asked the +// forge for the 20 most recently updated open pull requests of a repository, so a new head on the 21st or later was +// not announced and its merge check never started. A pass now pages through the open pull requests updated since its +// last look — every page on a sweep or the first pass — and stops at the first page that reaches back past the floor. + +const HOUR = 3600_000; +const T0 = Date.parse("2026-10-12T12:00:00Z"); +const sha = (n: number, v: string) => `${v}${String(n).padStart(4, "0")}`.padEnd(40, "0"); + +/** A forge holding `count` open pull requests of novox/hq, #n updated n hours after T0 (#count the newest), paged and + * sorted the way gitea answers `sort=recentupdate`: by update time, newest first. */ +function forge(count: number, heads: Record) { + const asked: URLSearchParams[] = []; + const server = createServer((req, res) => { + const url = new URL(req.url ?? "", "http://x"); + if (url.pathname !== "/api/v1/repos/novox/hq/pulls") { + res.statusCode = 404; + return res.end("{}"); + } + asked.push(url.searchParams); + const limit = Number(url.searchParams.get("limit") ?? 30); + const page = Number(url.searchParams.get("page") ?? 1); + const all = Array.from({ length: count }, (_, i) => count - i).map((n) => ({ + number: n, state: "open", base: { ref: "main" }, head: { ref: `b${n}`, sha: heads[n] ?? sha(n, "a") }, + updated_at: new Date(T0 + n * HOUR).toISOString(), html_url: `h/${n}`, + })); + res.end(JSON.stringify(all.slice((page - 1) * limit, page * limit))); + }); + return { server, asked }; +} + +async function serve(count: number, heads: Record) { + const { GiteaClient } = await import("../client.ts"); + const f = forge(count, heads); + await new Promise((r) => f.server.listen(0, r)); + const client = new GiteaClient(`http://127.0.0.1:${(f.server.address() as any).port}`, "t"); + return { client, ...f }; +} + +test("a new head beyond the newest 20 open pull requests is announced, exactly once", async () => { + const { headsToAnnounce } = await import("../pulls.ts"); + // 25 open, every head announced before; #3 — the 23rd most recently updated — got a new head. + const announced: Record = {}; + for (let n = 1; n <= 25; n++) announced[`novox/hq#${n}`] = sha(n, "a"); + const { client, server, asked } = await serve(25, { 3: sha(3, "b") }); + try { + // The fault: the newest 20 alone do not hold it. + const newest = await client.listPullRequests("novox", "hq", { state: "open", sort: "recentupdate", limit: "20" }); + assert.deepEqual(headsToAnnounce("novox/hq", newest, announced), [], "the newest 20 miss #3's new head"); + asked.length = 0; + const said: string[] = []; + for (let pass = 0; pass < 2; pass++) { + // The sweep, as index.ts's pollPulls asks: no floor, every page. + const open = await client.listOpenPullsSince("novox", "hq", ""); + for (const p of headsToAnnounce("novox/hq", open, announced)) { + said.push(`#${p.number}@${p.head_sha}`); + announced[`novox/hq#${p.number}`] = String(p.head_sha); + } + } + assert.deepEqual(said, [`#3@${sha(3, "b")}`], "the new head past the newest 20, announced once over two passes"); + assert.ok(asked.every((q) => q.get("state") === "open" && q.get("sort") === "recentupdate"), "asked newest first"); + } finally { + server.close(); + } +}); + +test("a pass pages until it reaches back past its floor, and no further", async () => { + // 130 open pull requests, #n updated n hours after T0: page 1 holds #130–#81, page 2 #80–#31, page 3 #30–#1. + const { client, server, asked } = await serve(130, {}); + try { + // A floor between #60 and #61: page 2 reaches back past it, so page 3 is never asked. + const floor = new Date(T0 + 60.5 * HOUR).toISOString(); + const open = await client.listOpenPullsSince("novox", "hq", floor); + assert.deepEqual(asked.map((q) => q.get("page")), ["1", "2"]); + assert.equal(open.length, 100); + assert.ok(open.some((p) => p.number === 61), "every one updated since the floor is read"); + // A floor no page reaches past: the last short page ends it. + asked.length = 0; + assert.equal((await client.listOpenPullsSince("novox", "hq", "")).length, 130); + assert.deepEqual(asked.map((q) => q.get("page")), ["1", "2", "3"]); + // Every page holds fewer than 50 updated since a recent floor: one page. + asked.length = 0; + await client.listOpenPullsSince("novox", "hq", new Date(T0 + 129 * HOUR).toISOString()); + assert.deepEqual(asked.map((q) => q.get("page")), ["1"]); + } finally { + server.close(); + } +}); diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 3bbfc5e9..750c8122 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -414,7 +414,8 @@ test("the tools register once there is a way to a token, and the first call mint [ "gitea_list_repos", "gitea_create_repo", "gitea_delete_repo", "gitea_list_issues", "gitea_get_issue", "gitea_create_issue", "gitea_close_issue", "gitea_add_comment", - "gitea_list_pull_requests", "gitea_get_pull_request", "gitea_create_pull_request", "gitea_merge_pull_request", + // gitea_merge_pull_request is served from the watcher's bundle, beside the merge check's one writer (ADR 0323). + "gitea_list_pull_requests", "gitea_get_pull_request", "gitea_create_pull_request", "gitea_close_pull_request", "gitea_reopen_pull_request", "gitea_update_pull_request", diff --git a/modules/gitea/test/trunk.test.ts b/modules/gitea/test/trunk.test.ts new file mode 100644 index 00000000..7e7fdd4b --- /dev/null +++ b/modules/gitea/test/trunk.test.ts @@ -0,0 +1,175 @@ +// **A merge check's verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341): both statuses name +// the head and the trunk commit; a trunk's move puts the merge check of every open pull request into that trunk — and +// of none into another — back to pending with the reason, and runs nothing; a verdict that lands judged against a trunk +// that has moved since is said out of date. + +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { checkOnItsWay, lateVerdictComment, outOfDateStatuses, outOfDateWords, trunkNamed, verdictAtTrunk, weighTrunks } from "../dist/trunk.js"; +import { GateWriter } from "../dist/gate.js"; +import { statusesFor, commentFor } from "../dist/pulls.js"; + +const HEAD = "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const TRUNK = "a2c2c35bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const MOVED = "d28ca95ccccccccccccccccccccccccccccccccc"; + +test("both statuses say the head and the trunk commit the verdict judged, first, whatever the summary's length", () => { + const c = { owner: "novox", repo: "mesh-host", number: 66, commit: HEAD, verdict: "pass", summary: "x", id: "build-1", + trunk: TRUNK.slice(0, 12), beside: { "mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210" }, + gate: { verdict: "pass", summary: "4 of 4 compose ".repeat(20) }, + "repo-check": { verdict: "fail", summary: "its merge-check.sh failed: internal/store/store.go:145:2: Declared redeclared" } }; + const [gate, repo] = statusesFor(c); + for (const s of [gate, repo]) { + assert.ok(s.description.startsWith("head dfb1415aaaaa merged into trunk a2c2c35bbbbb — "), s.description); + assert.ok(s.description.length <= 140, `${s.description.length} characters`); + assert.equal(trunkNamed(s.description), "a2c2c35bbbbb"); + } + assert.equal(gate.state, "success"); + assert.equal(repo.state, "failure"); + const comment = commentFor(c)!; + assert.match(comment, /Judged: head `dfb1415aaaaa` merged into trunk `a2c2c35bbbbb`; read beside it: mesh-catalog `0123456789ab`, mesh-lab `ba9876543210`/); + + // A verdict of a build seat from before the rule names no trunk, and is said as before. + const [old] = statusesFor({ ...c, trunk: undefined, gate: { verdict: "pass", summary: "every machine composes" } }); + assert.equal(old.description, "pass: every machine composes"); + assert.equal(trunkNamed(old.description), undefined); +}); + +test("a trunk's move puts an open head's merge check out of date, naming both commits, unless a check is on its way", () => { + const verdict = (trunk: string, state = "success") => ({ + "mesh/merge-gate": { state, description: `head ${HEAD.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: x` }, + "mesh/repo-check": { state, description: `head ${HEAD.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: y` } }); + const put = outOfDateStatuses(verdict(TRUNK), MOVED); + assert.deepEqual(put.map((s) => [s.context, s.state]), [["mesh/merge-gate", "pending"], ["mesh/repo-check", "pending"]]); + assert.equal(put[0].description, "out of date: judged against trunk a2c2c35bbbbb, the trunk is now d28ca95ccccc; " + + "judged again on a recheck or when it is merged"); + assert.equal(put[0].description, outOfDateWords(TRUNK, MOVED)); + assert.equal(trunkNamed(put[0].description), "a2c2c35bbbbb", "an out-of-date status still names what it judged"); + // A failure is out of date too: it judged another merge. + assert.equal(outOfDateStatuses(verdict(TRUNK, "failure"), MOVED).length, 2); + // Already judged against the trunk's head; a check on its way or already out of date; no check yet: left alone. + assert.deepEqual(outOfDateStatuses(verdict(MOVED), MOVED), []); + assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": { state: "pending", description: "building on ace since 14:02" } }, MOVED), []); + assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": { state: "pending", description: put[0].description } }, MOVED), []); + assert.deepEqual(outOfDateStatuses({}, MOVED), []); + // A head with no repository check of its own gets no pending one a verdict would never set. + assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": verdict(TRUNK)["mesh/merge-gate"] }, MOVED).map((s) => s.context), + ["mesh/merge-gate"]); + // A verdict naming no trunk, of a build seat from before the rule, judged an earlier trunk. + assert.match(outOfDateStatuses({ "mesh/merge-gate": { state: "success", description: "pass: x" } }, MOVED)[0].description, + /judged against a trunk it does not name, the trunk is now d28ca95ccccc/); + + assert.ok(checkOnItsWay({ state: "pending", description: "waiting in the build queue since 14:00" })); + assert.ok(checkOnItsWay({ state: "pending", description: "checking again: a person asked" })); + assert.ok(!checkOnItsWay({ state: "pending", description: put[0].description })); + assert.deepEqual(verdictAtTrunk(verdict(MOVED), MOVED), { current: true, onItsWay: false, judged: "d28ca95ccccc" }); + assert.equal(verdictAtTrunk(verdict(TRUNK), MOVED).current, false); +}); + +/** A forge holding open pull requests into two branches, each head's statuses, and what was written. */ +function aForge(pulls: { number: number; base: string; head: string }[], heads: Record) { + const statuses = new Map>(); + const writes: string[] = []; + const verdict = (trunk: string) => ({ + "mesh/merge-gate": { state: "success", description: `head x merged into trunk ${trunk.slice(0, 12)} — pass` }, + "mesh/repo-check": { state: "success", description: `head x merged into trunk ${trunk.slice(0, 12)} — pass` } }); + for (const p of pulls) statuses.set(p.head, verdict(heads[p.base])); + const forge = { + listOpenPulls: async () => pulls.map((p) => ({ number: p.number, title: "t", state: "open", merged: false, base: p.base, + head: "feat", head_sha: p.head, html_url: `http://forge/pulls/${p.number}` })), + branchHead: async (_o: string, _r: string, branch: string) => heads[branch], + setCommitStatus: async (_o: string, _r: string, sha: string, s: { context: string; state: string; description: string }) => { + writes.push(`${sha} ${s.context} ${s.state}`); + statuses.set(sha, { ...statuses.get(sha), [s.context]: { state: s.state, description: s.description } }); + }, + gateOf: async () => undefined, + gateHistory: async () => [], + headChecks: async (_o: string, _r: string, sha: string) => statuses.get(sha) ?? {}, + }; + return { forge, writes, statuses }; +} + +test("a push to a trunk puts every open pull request into it out of date — twenty-five of them — and none into another", async () => { + const heads: Record = { main: TRUNK, release: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }; + const pulls = Array.from({ length: 25 }, (_, i) => ({ number: i + 1, base: "main", head: `${String(i).padStart(2, "0")}`.repeat(20) })); + pulls.push({ number: 99, base: "release", head: "9".repeat(40) }); + const { forge, writes, statuses } = aForge(pulls, heads); + const gates = new GateWriter(forge); + const trunks: Record = {}; + + // First sight: each trunk is taken as it is, and nothing is put out of date. + assert.equal(await weighTrunks(forge, gates, trunks, "novox", "mesh-host"), true); + assert.deepEqual(writes, []); + assert.equal(trunks["novox/mesh-host:main"], TRUNK); + // Nothing moved: nothing read past the trunks, nothing written. + assert.equal(await weighTrunks(forge, gates, trunks, "novox", "mesh-host"), false); + + heads.main = MOVED; + assert.equal(await weighTrunks(forge, gates, trunks, "novox", "mesh-host"), true); + assert.equal(writes.length, 50, "both statuses of each of the 25 pull requests into main"); + for (const p of pulls.filter((p) => p.base === "main")) { + const s = statuses.get(p.head)!; + assert.equal(s["mesh/merge-gate"].state, "pending"); + assert.match(s["mesh/merge-gate"].description!, /judged against trunk a2c2c35bbbbb, the trunk is now d28ca95ccccc/); + assert.equal(s["mesh/repo-check"].state, "pending"); + } + assert.equal(statuses.get("9".repeat(40))!["mesh/merge-gate"].state, "success", "a pull request into another branch is left"); + assert.equal(trunks["novox/mesh-host:main"], MOVED); + + // Weighed again, nothing more: the move was taken once, and the out-of-date statuses are left as they are. + writes.length = 0; + heads.main = "f".repeat(40); + await weighTrunks(forge, gates, trunks, "novox", "mesh-host"); + assert.deepEqual(writes, [], "a check already out of date is not written again"); +}); + +test("a trunk that cannot be read is weighed again on the next pass, never recorded half", async () => { + const heads: Record = { main: TRUNK }; + const { forge } = aForge([{ number: 1, base: "main", head: "1".repeat(40) }], heads); + const trunks: Record = { "novox/mesh-host:main": TRUNK }; + heads.main = MOVED; + const failing = { ...forge, setCommitStatus: async () => { throw new Error("the forge is away"); } }; + assert.equal(await weighTrunks(failing, new GateWriter(failing), trunks, "novox", "mesh-host"), false); + assert.equal(trunks["novox/mesh-host:main"], TRUNK, "not recorded: the next pass weighs it again"); +}); + +test("the trunk's move takes its turn after a verdict in flight, and weighs what the forge holds then", async () => { + const heads: Record = { main: TRUNK }; + const { forge, statuses } = aForge([{ number: 1, base: "main", head: HEAD }], heads); + const gates = new GateWriter(forge); + // A verdict judged against the trunk's new head, in flight when the move is weighed: it lands first, and is left. + let release!: () => void; + const held = new Promise((r) => (release = r)); + const verdictLanded = gates.verdict("novox", "mesh-host", HEAD, async () => { + await held; + statuses.set(HEAD, { "mesh/merge-gate": { state: "success", description: `head x merged into trunk ${MOVED.slice(0, 12)} — pass` } }); + }); + const moved = gates.outOfDate("novox", "mesh-host", HEAD, (checks) => outOfDateStatuses(checks, MOVED)); + release(); + await verdictLanded; + assert.deepEqual(await moved, [], "a verdict judged against the trunk's head is not put out of date"); + assert.equal(statuses.get(HEAD)!["mesh/merge-gate"].state, "success"); +}); + +test("a verdict judged against a trunk newer than the pass's first read is not put out of date", async () => { + const NEWER = "ffffffffffffffffffffffffffffffffffffffff"; + const heads: Record = { main: TRUNK }; + const { forge, statuses } = aForge([{ number: 1, base: "main", head: HEAD }], heads); + const trunks: Record = { "novox/mesh-host:main": TRUNK }; + // The pass reads main at MOVED; by the head's turn the trunk moved again, and the head's check answered for it. + let reads = 0; + const moving = { ...forge, branchHead: async () => (reads++ === 0 ? MOVED : NEWER) }; + statuses.set(HEAD, { "mesh/merge-gate": { state: "success", description: `head x merged into trunk ${NEWER.slice(0, 12)} — pass` } }); + await weighTrunks(moving, new GateWriter(moving), trunks, "novox", "mesh-host"); + assert.equal(statuses.get(HEAD)!["mesh/merge-gate"].state, "success", "a fresh verdict was put out of date"); +}); + +test("a verdict that landed out of date keeps its account on the pull request", () => { + const words = outOfDateWords(TRUNK, MOVED); + const own = lateVerdictComment({ commit: HEAD, trunk: TRUNK.slice(0, 12), verdict: "fail" }, "**Merge check** at `dfb1415a`\n\nthe report", words); + assert.match(own, /the report/); + assert.match(own, /\*\*Out of date when it landed\*\*: out of date: judged against trunk a2c2c35bbbbb/); + const bare = lateVerdictComment({ commit: HEAD, trunk: TRUNK.slice(0, 12), verdict: "pass" }, null, words); + assert.match(bare, /head dfb1415aaaaa merged into trunk a2c2c35bbbbb, pass/); +}); diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index ede929c7..201b08d0 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -63,6 +63,8 @@ export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] { return namesChecked(allGiteaTools(gitea)); } +// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one +// writer (novox/hq issue 456, ADR 0323). function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { return [ // ---- Repositories ---- @@ -263,111 +265,6 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { }), }), }, - { - name: "gitea_merge_pull_request", - description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head " + - "and every status its base branch's protection requires succeeded there (mesh/merge-gate, mesh/repo-check), " + - "optionally deleting the source branch afterwards. Refused otherwise, with nothing merged: no tool merges past " + - "a check, and none merges a head that moved after it was reviewed.", - input: { - owner: { type: "string", description: "the repository owner" }, - repo: { type: "string", description: "the repository name" }, - number: { type: "number", description: "the PR number" }, - head: { type: "string", description: "the commit you reviewed, as its sha or at least its first 12 characters: " + - "refused unless it is the pull request's head" }, - method: { type: "string", description: "merge (default) | fast-forward-only | squash | rebase | rebase-merge. " + - "Only merge and fast-forward-only can be read back to prove the head you named is what landed" }, - delete_branch: { type: "boolean", description: "delete the source branch after merge (default true)" }, - }, - run: async (args) => { - const owner = String(args.owner); - const repo = String(args.repo); - const number = Number(args.number); - const method = args.method ? String(args.method) : "merge"; - if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`); - const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch); - // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left - // to the forge, whose administrator — the account this tool's token is minted for — may merge past a - // rule that lets it. A rule that cannot be read is no rule passed. - // **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d, - // force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the - // head was and nothing told it which head the review had approved. The caller names it; a head that moved - // since is refused, and the forge is handed the same head, so a push between this read and the merge is - // refused by the forge too. - const named = args.head === undefined ? "" : String(args.head).trim().toLowerCase(); - if (!named) { - throw new Refused(`${owner}/${repo}#${number} is not merged: name the head you reviewed (head), so that only ` + - `that commit can land (novox/hq issue 381). Nothing was merged`); - } - // Twelve characters, not seven: an author who force-pushes can make a commit whose first seven match the - // reviewed one in seconds, and the review would then merge it. Twelve cannot be matched that way. - if (!REVIEWED_HEAD.test(named)) { - throw new Refused(`${owner}/${repo}#${number} is not merged: head ${named} does not name one commit — give its ` + - `sha, or at least its first 12 characters. Nothing was merged`); - } - const pull = await gitea.getPullRequest(owner, repo, number); - if (pull.merged) throw new Refused(`${owner}/${repo}#${number} is already merged`); - if (!pull.head_sha || !pull.base) throw new Refused(`${owner}/${repo}#${number} names no head commit or base branch`); - if (!pull.head_sha.toLowerCase().startsWith(named)) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` + - `${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`); - } - const rule = await gitea.branchProtection(owner, repo, pull.base); - // Only into a branch whose protection requires the checks: a pull request whose base was moved to a - // branch nothing guards would merge past every check (novox/hq ADR 0266). - if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) { - throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule requiring ` + - `the checks guards: a tool merges only into such a branch (novox/hq ADR 0266). Nothing was merged`); - } - const statuses = newestByContext((await gitea.combinedStatus(owner, repo, pull.head_sha)).statuses); - const verdict = mergeableUnder(pull.base, rule, statuses); - if (verdict.mergeable !== true) { - throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` + - `its checks succeed; no tool merges past them (novox/hq ADR 0266)`); - } - await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha); - // The merge commit only exists now; answered so the caller can follow what is built from it. - // pull.merged is the events entrypoint's to announce (index.ts), once, within its poll. - // From here the merge has happened. Nothing below may read as if it had not: a read-back that fails or cannot - // be done is said beside merged: true, and only a proven other head is an error — one that says the merge happened. - const answer: Record = { merged: true, number, method, head: pull.head_sha, deleted_branch: deleteBranch }; - let landed: string | undefined; - try { - const merged = await gitea.getPullRequest(owner, repo, number); - answer.merge_commit_sha = merged.merge_commit_sha; - // What landed, read back from the commit the merge made rather than from the forge's record of the pull - // request, which is the very thing a race would leave behind. A merge commit's second parent is the head it - // joined; a fast-forward lands the head itself. A squash or a rebase makes new commits that name no head. - if (!merged.merge_commit_sha) { - answer.read_back = "the forge named no merge commit, so what landed cannot be read back"; - } else if (method === "merge") { - const parents = await gitea.commitParents(owner, repo, merged.merge_commit_sha); - if (parents.length === 2) landed = parents[1]; - else answer.read_back = `the merge commit has ${parents.length} parent(s), not two: what landed cannot be read back`; - } else if (method === "fast-forward-only") { - // That the forge names the head itself as a fast-forward's merge commit is assumed, not seen on this forge: - // a different sha is said, not raised as another head having landed. - if (merged.merge_commit_sha === pull.head_sha) landed = merged.merge_commit_sha; - else answer.read_back = `the forge named ${merged.merge_commit_sha.slice(0, 12)} as this fast-forward's commit, ` + - `not the head: that the head landed could not be confirmed`; - } else { - answer.read_back = `a ${method} makes new commits that name no head: what landed cannot be read back`; - } - } catch (err) { - answer.read_back = `the merge happened, but reading back what landed failed: ${(err as Error).message}`; - } - // Loud, not a field beside merged: true that a caller reading only that would miss. The merge cannot be taken - // back by this tool; what must not happen is that it reads as the merge that was asked for. - if (landed !== undefined && landed !== pull.head_sha) { - throw new Error(`${owner}/${repo}#${number} WAS merged (${answer.merge_commit_sha}), but the forge merged ` + - `${landed.slice(0, 12)}, not the ${pull.head_sha.slice(0, 12)} it was handed and you reviewed: a commit nobody ` + - `reviewed has landed. Read it before anything is built from it (novox/hq issue 381)`); - } - if (landed !== undefined) answer.landed = landed; - return answer; - }, - }, - { name: "gitea_close_pull_request", description: "Close a pull request without merging it — one whose work landed elsewhere, or was abandoned.", diff --git a/modules/gitea/trunk.ts b/modules/gitea/trunk.ts new file mode 100644 index 00000000..1f0b1a6a --- /dev/null +++ b/modules/gitea/trunk.ts @@ -0,0 +1,250 @@ +// **A merge check's verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341). +// +// The build seat checks a pull request's head merged into its trunk's head at that moment — the verdict's trunk +// commit — and the verdict names both: "head 1a2b3c4d5e6f merged into trunk 7a8b9c0d1e2f", in both statuses' +// descriptions. When the trunk moves, every open pull request's verdict into it is out of date: +// +// - **the forge's holder** (index.ts) hears the move on its own — it reads each trunk's head on every pass that +// looks at the repository, never through mesh-delivery (ADR 0239 §36) — and sets mesh/merge-gate and +// mesh/repo-check on the head of every open pull request into that trunk back to pending, with the reason. +// Nothing is run for it: a recheck, a new head, or the merge tool asks the check again; +// - **the merge tool** (tools/index.ts) compares the trunk commit the head's verdict names with the trunk's head at +// the moment it is asked, merges nothing when they differ, asks the head's check once — the same announcement a +// new head makes — and says how long the last check of that repository took. It never merges by itself after. +// +// Pure functions and two small files of the module's own state, so this is tested without a forge. + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import type { CommitStatus, GiteaClient, GiteaPull, HeadCheck } from "./client.js"; +import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js"; + +/** A commit as a verdict names it: its first twelve characters. */ +export function twelve(sha: string | undefined): string { + return String(sha ?? "").slice(0, 12).toLowerCase(); +} + +/** Whether two commits are one, however either is abbreviated (seven characters at least). */ +export function sameCommit(a: string | undefined, b: string | undefined): boolean { + const x = String(a ?? "").toLowerCase(), y = String(b ?? "").toLowerCase(); + if (x.length < 7 || y.length < 7) return false; + return x.length <= y.length ? y.startsWith(x) : x.startsWith(y); +} + +/** What a verdict judged, as both statuses' descriptions begin (ADR 0323 §3). */ +export function judgedWords(head: string, trunk: string): string { + return `head ${twelve(head)} merged into trunk ${twelve(trunk)}`; +} + +/** The trunk commit a merge check's status names: the one its verdict judged ("merged into trunk …"), or, for a + * verdict put out of date, the one it had judged ("judged against trunk …"). Undefined when it names none, as a + * verdict of a build seat from before ADR 0323 does. */ +export function trunkNamed(description: string | undefined): string | undefined { + const m = /(?:merged into|judged against) trunk ([0-9a-f]{7,64})\b/i.exec(description ?? ""); + return m ? m[1].toLowerCase() : undefined; +} + +/** A verdict out of date, as its statuses say it (ADR 0323 §4). */ +export function outOfDateWords(judged: string | undefined, now: string): string { + const was = judged ? `judged against trunk ${twelve(judged)}` : "judged against a trunk it does not name"; + return `out of date: ${was}, the trunk is now ${twelve(now)}; judged again on a recheck or when it is merged`; +} + +/** Whether a pending gate says a check of the head is asked and not yet answered: waiting in the build queue, + * building on a machine (gate.ts), or asked again by a recheck (mesh-delivery). Such a check is not asked again. */ +export function checkOnItsWay(gate: HeadCheck | undefined): boolean { + if (!gate || gate.state !== "pending") return false; + const d = gate.description ?? ""; + return d.startsWith("waiting in the build queue") || d.startsWith("building on ") || d.startsWith("checking again"); +} + +/** + * The statuses a trunk's move puts back to pending on one open pull request's head, from what the forge holds on it + * now: none when it has no merge check yet, when a check is pending (on its way, or already out of date — its + * verdict is weighed against the trunk when it lands), or when its verdict judged the trunk's head. Otherwise the + * gate, and the repository's own check when the head has one — a pending repo-check a verdict will not set would + * stand for ever (novox/hq issue 305's rule). + */ +export function outOfDateStatuses(checks: Record, trunkHead: string): CommitStatus[] { + const gate = checks[CHECK_CONTEXT]; + if (!gate || gate.state === "pending") return []; + const judged = trunkNamed(gate.description); + if (judged && sameCommit(judged, trunkHead)) return []; + const description = outOfDateWords(judged, trunkHead); + const out: CommitStatus[] = [{ state: "pending", context: CHECK_CONTEXT, description }]; + if (checks[REPO_CHECK_CONTEXT]) out.push({ state: "pending", context: REPO_CHECK_CONTEXT, description }); + return out; +} + +/** Where the merge tool stands with a head's verdict and the trunk's head (ADR 0323 §5). */ +export interface VerdictAtTrunk { + /** The verdict judged the trunk's head: the required statuses decide, as before. */ + current: boolean; + /** A check of the head is asked and not yet answered: nothing is asked again. */ + onItsWay: boolean; + /** The trunk commit the verdict names, when it names one. */ + judged?: string; +} + +export function verdictAtTrunk(checks: Record, trunkHead: string): VerdictAtTrunk { + const gate = checks[CHECK_CONTEXT]; + const judged = trunkNamed(gate?.description); + if (checkOnItsWay(gate)) return { current: false, onItsWay: true, judged }; + return { current: !!gate && gate.state !== "pending" && !!judged && sameCommit(judged, trunkHead), onItsWay: false, judged }; +} + +/** The pending words the merge tool's ask of a check is said with: in the build queue from now, as an announced head's. */ +export function askedWords(at: Date, trunkHead: string): string { + const two = (n: number) => String(n).padStart(2, "0"); + return `waiting in the build queue since ${two(at.getHours())}:${two(at.getMinutes())}: asked by the merge tool ` + + `against trunk ${twelve(trunkHead)}`; +} + +/** The comment a verdict that landed out of date leaves (ADR 0323 §7: kept and shown): its own account when it has + * one, else what it judged and said, and that it is out of date. */ +export function lateVerdictComment(c: { commit: string; trunk?: string; verdict: string }, account: string | null, + outOfDate: string): string { + const own = account ?? `**Merge check** at \`${c.commit.slice(0, 8)}\`: ${judgedWords(c.commit, c.trunk ?? "")}, ${c.verdict}`; + return `${own}\n\n**Out of date when it landed**: ${outOfDate}`; +} + +// --- a trunk's move, weighed -------------------------------------------------------------------------------------- + +/** What weighing a trunk's move reads of the forge. */ +export interface TrunkForge { + listOpenPulls(owner: string, repo: string): Promise; + branchHead(owner: string, repo: string, branch: string): Promise; +} + +/** What it writes with: the merge check's one writer (gate.ts), which decides in the head's own turn. */ +export interface OutOfDateWriter { + outOfDate(owner: string, repo: string, sha: string, + decide: (checks: Record) => CommitStatus[] | Promise, target?: string): Promise; +} + +/** + * Every trunk of one repository that an open pull request merges into, read; for one that moved since it was last read + * (`trunks`, by `owner/repo:branch`, which this keeps current), the merge check of every open pull request into it put + * out of date (ADR 0323 §4). A trunk first seen is recorded as it is and nothing is put out of date. Bounded however + * many are open: one listing of the open pull requests (paged), one read of each trunk, then per open pull request into + * a trunk that moved one read and at most two writes, one at a time. A trunk that could not be weighed whole is not + * recorded, so the next pass weighs it again. Answers whether the record changed. + */ +export async function weighTrunks(forge: TrunkForge, gates: OutOfDateWriter, trunks: Record, owner: string, + name: string): Promise { + let changed = false; + try { + const open = (await forge.listOpenPulls(owner, name)).filter((p) => p.state === "open" && !!p.head_sha && !!p.base); + for (const base of new Set(open.map((p) => String(p.base)))) { + const key = `${owner}/${name}:${base}`.toLowerCase(); + const head = await forge.branchHead(owner, name, base); + const before = trunks[key]; + if (before && sameCommit(before, head)) continue; + if (before) { + let put = 0; + for (const pull of open.filter((p) => p.base === base)) { + // The trunk is read again in the head's own turn: a verdict judged against a head newer than the pass's first + // read — the trunk moved again meanwhile, and a check answered — is not put out of date. + const written = await gates.outOfDate(owner, name, String(pull.head_sha), + async (checks) => outOfDateStatuses(checks, await forge.branchHead(owner, name, base)), pull.html_url || undefined); + if (written.length) put++; + } + console.log(`[gitea] ${owner}/${name}'s ${base} moved from ${twelve(before)} to ${twelve(head)}: ${put} open pull ` + + `request(s) into it put out of date, nothing run (novox/hq ADR 0323)`); + } + trunks[key] = head; + changed = true; + } + } catch (err) { + console.error(`[gitea] ${owner}/${name}: whether a trunk moved could not be weighed, so it is weighed again on the next ` + + `pass — ${err instanceof Error ? err.message : err}`); + } + return changed; +} + +// --- how long the last check of a repository took ------------------------------------------------------------- + +/** How long one repository's last check took, and when it was heard: kept by the watcher, read by the merge tool. */ +export interface Took { + took: string; + at: string; + head?: string; +} + +/** The file the watcher keeps each repository's last check's duration in, beside its other records. */ +export function tookFile(stateDir: string | undefined): string | null { + return stateDir ? `${stateDir.replace(/\/+$/, "")}/checks-took.json` : null; +} + +function readTookAll(file: string | null): Record { + if (!file || !existsSync(file)) return {}; + try { + return JSON.parse(readFileSync(file, "utf8")) as Record; + } catch { + return {}; + } +} + +/** Keep how long a repository's check took. A record that cannot be written is said, never fatal to the verdict. */ +export function keepTook(file: string | null, repository: string, took: Took): void { + if (!file || !took.took) return; + try { + const all = readTookAll(file); + all[repository.toLowerCase()] = took; + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file + ".tmp", JSON.stringify(all)); + renameSync(file + ".tmp", file); + } catch (err) { + console.error(`[gitea] how long ${repository}'s check took could not be kept — ${err instanceof Error ? err.message : err}`); + } +} + +/** How long a repository's last check took, in words for the merge tool's refusal. */ +export function tookWords(file: string | null, repository: string): string { + const t = readTookAll(file)[repository.toLowerCase()]; + if (!t) return `how long a check of ${repository} takes is not known yet: no verdict of it was heard since this was kept`; + return `the last check of ${repository} took ${t.took} (heard ${t.at})`; +} + +// --- the announcement of a head, shared by the watcher and the merge tool --------------------------------------- + +/** `pull.updated` for one open pull request's head: what the controller asks the build seat to check (novox/hq to-be + * 45 §9). The merge tool's ask is this same announcement (ADR 0323 §5). */ +export async function announcementOf(client: GiteaClient, owner: string, name: string, cloneURL: string | undefined, + pull: GiteaPull): Promise> { + const files = await client.listPullFiles(owner, name, pull.number); + const head = String(pull.head_sha); + // What the controller maps the change onto the mesh's module graph with (novox/hq ADR 0237 as amended): which + // changed directories hold a module at the head — the merge's rule (issue 278) — and whether the head holds the + // repository's own merge-check.sh. Not said when it could not be read; the controller then reads the change as + // touching everything built from the repository. + let moduleDirs: string[] | null = null; + let mergeCheck: boolean | null = null; + try { + if (!files.truncated) moduleDirs = await client.moduleDirsAt(owner, name, head, files.paths); + mergeCheck = await client.holdsFile(owner, name, head, "merge-check.sh"); + } catch (err) { + console.error(`[gitea] ${owner}/${name}#${pull.number}: what the head holds could not be read — ${err instanceof Error ? err.message : err}`); + } + // The base branch's head as the head is announced: the trunk commit a verdict no build judges is said against + // (novox/hq ADR 0323 §3). Not said when it cannot be read; that verdict then names none. + const baseSHA = pull.base ? await client.branchHead(owner, name, String(pull.base)).catch(() => undefined) : undefined; + return { + owner, + repo: name, + number: pull.number, + ...(baseSHA ? { base_sha: baseSHA } : {}), + title: pull.title, + body: pull.body, + base: pull.base, + head: pull.head, + head_sha: pull.head_sha, + clone_url: cloneURL, + html_url: pull.html_url, + paths: files.paths, + paths_truncated: files.truncated, + removed: files.removed, + ...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}), + ...(mergeCheck !== null ? { merge_check: mergeCheck, merge_check_said: true } : {}), + }; +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/checks.go b/modules/mesh-delivery/cmd/mesh-delivery/checks.go index dadbdead..a631a5dd 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/checks.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/checks.go @@ -51,6 +51,10 @@ type ChecksAnswer struct { Statuses []CheckStatus `json:"statuses"` // Verdict is the merge check's verdict whole, as this owner heard it for this commit. Verdict *Verdict `json:"verdict,omitempty"` + // Judged is what the verdict judged: the head merged into the trunk commit (novox/hq ADR 0323 §3); OutOfDate + // why it no longer counts, when its trunk has moved since. + Judged string `json:"judged,omitempty"` + OutOfDate string `json:"out_of_date,omitempty"` // Merge is whether the branch's protection lets the commit merge; Mergeable nil when it is not known. Merge ForgeMerge `json:"merge"` // Forge is why the forge could not be read, when it could not. @@ -118,6 +122,7 @@ func (h *Holder) Checks(repository string, number int, commit string) (*ChecksAn } if d.Check != nil && sameCommit(d.Commit, a.Commit) { a.Verdict = d.Check + a.Judged, a.OutOfDate = d.Check.Judged(), d.outOfDate() } } @@ -273,6 +278,12 @@ func (a *ChecksAnswer) says() string { default: merge = "NOT mergeable — " + merge } + if a.Judged != "" { + what += " (" + a.Judged + ")" + } + if a.OutOfDate != "" { + merge += "; its verdict is " + a.OutOfDate + } return what + ": " + statuses + "; " + merge } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go b/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go index ef43241b..ca989ff1 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go @@ -451,7 +451,7 @@ func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) { State: Ready, Since: t0, Created: t0.Add(-time.Hour), Plan: planMoving(b, "b"), Check: &Verdict{Gate: "pass"}} g := &Group{ID: "feat/k", Members: []string{aID, bID}, Order: []string{aID, bID}, Closed: true, OrderOf: map[string]string{aID: aHead, bID: bHead}, - Check: &GroupCheck{Asked: "check-1", At: t0, Commits: c.commits}} + Check: &GroupCheck{Asked: "check-old", At: t0, Commits: c.commits}} for _, d := range []*Delivery{da, db} { _ = store.PutDelivery(d) } @@ -463,7 +463,7 @@ func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) { if s := w.state(aID); s != Held { t.Fatalf("before its composed verdict it is %s", s) } - w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-1", Verdict: c.verdict, Summary: "composed: " + c.verdict}) + w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-old", Verdict: c.verdict, Summary: "composed: " + c.verdict}) w.settleAll() if s := w.state(aID); s != c.want { t.Fatalf("its composed check %s, and it is %s (%s), want %s", c.name, s, w.delivery(aID).HeldWhy, c.want) @@ -471,6 +471,10 @@ func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) { if c.want == Delivering && !containsString(ctl.delivers, "plan-a") { t.Fatalf("lifted, its walk was not let go: %v", ctl.delivers) } + // A composed check of other heads is asked again for these heads (novox/hq ADR 0323 §6). + if c.name == "passed for other heads" && !containsString(ctl.checks, "feat/k:"+aID+","+bID) { + t.Fatalf("the composed check was not asked again for the heads that merged: %v", ctl.checks) + } }) } } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index fe055b0d..81c8279d 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -280,6 +280,8 @@ type PullEvent struct { ModuleDirs []string `json:"module_dirs,omitempty"` ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` HeadChecks map[string]string `json:"head_checks,omitempty"` + // BaseSHA is the base branch's head when the head was announced (novox/hq ADR 0323). + BaseSHA string `json:"base_sha,omitempty"` } func (p PullEvent) repository() string { return strings.ToLower(p.Owner + "/" + p.Repo) } @@ -297,10 +299,22 @@ func (h *Holder) PullUpdated(p PullEvent) { h.mu.Lock() defer h.mu.Unlock() id := IDOf(p.repository(), p.HeadSHA) - if d := h.deliveries[id]; d != nil { - return // announced again: nothing new - } now := h.Now() + if d := h.deliveries[id]; d != nil { + // Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its + // check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound + // runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it. + if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA { + d.BaseSHA = p.BaseSHA + h.keep(d) + } + if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil { + d.AskedAgain = &now + h.keep(d) + } + return + } + replaced := "" for _, old := range h.deliveries { if old.Repository == p.repository() && old.Number == p.Number && !sameCommit(old.Commit, p.HeadSHA) && !old.State.Final() && old.MergedAs == "" { @@ -308,13 +322,14 @@ func (h *Holder) PullUpdated(p PullEvent) { h.settleAndKeep(old) if old.Group != "" { h.leaveGroup(old) + replaced = old.ID } } } d := &Delivery{ID: id, Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid, - After: AfterIn(p.Body), Created: now} + After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA} t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)}) if err != nil { h.refuse(err) @@ -328,7 +343,45 @@ func (h *Holder) PullUpdated(p PullEvent) { ts = append(ts, settle(d, Facts{Now: now})...) } h.moved(d, ts) - h.joinGroup(d) + if !h.takePlaceOf(replaced, d) { + h.joinGroup(d) + } +} + +// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded, +// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The +// new head has its own check, and the group's composed check runs again for the new heads rather than holding the +// group for a person (novox/hq ADR 0323 §6, issue 406). An open group takes it as joinGroup does. +func (h *Holder) takePlaceOf(old string, d *Delivery) bool { + if old == "" { + return false + } + for _, g := range h.groups { + _, ordered := g.OrderOf[old] + if !g.Closed || !ordered && !containsString(g.Order, old) { + continue + } + if ordered { + delete(g.OrderOf, old) + g.OrderOf[d.ID] = d.Commit + } + if !containsString(g.Members, d.ID) { + g.Members = append(g.Members, d.ID) + sort.Strings(g.Members) + } + for i, id := range g.Order { + if id == old { + g.Order[i] = d.ID + } + } + d.Group = g.ID + g.Updated = h.Now() + h.keep(d) + h.keepGroup(g) + h.Logf("[mesh-delivery] group %s: %s takes the place of %s, which it superseded", g.ID, d.ID, old) + return true + } + return false } // decidedOnHead is the verdict the forge's statuses on a head hold, when the gate decided one: passed, warned @@ -469,6 +522,47 @@ func (h *Holder) PullMerged(p PullEvent) { } h.moved(d, ts) h.closeGroupOf(d) + h.trunkMoved(d, p.Base, p.MergeCommit, now) +} + +// trunkMoved takes a merge into a trunk as that trunk's move (novox/hq ADR 0323 §7): every other open delivery into +// it keeps the trunk's new head, and a ready one whose verdict names the commit before goes back to proposed. A +// group's composed check goes out of date when the merge is not one of its members and a member still open merges +// into that trunk (§6); the group then reads checking. Nothing is asked: a recheck or the merge tool judges again. +func (h *Holder) trunkMoved(merged *Delivery, base, commit string, now time.Time) { + if base == "" || commit == "" { + return + } + for _, o := range h.deliveries { + if o == merged || o.Repository != merged.Repository || o.Base != base || o.MergedAs != "" || o.State.Final() || + sameCommit(o.TrunkNow, commit) { + continue + } + // A check asked again against the trunk before this move judged that trunk: it is out of date too. + o.TrunkNow, o.AskedAgain = commit, nil + h.settleAndKeep(o) + } + for _, g := range h.groups { + if g.ID == merged.Group || g.Check == nil || g.Check.Verdict == "" || g.Check.OutOfDate != "" { + continue + } + for _, m := range h.membersOf(g) { + if m.Repository != merged.Repository || m.Base != base || m.MergedAs != "" || m.State.Final() { + continue + } + judged := g.Check.Trunks[m.ID] + if judged == "" { + judged = "?" + } + g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was judged against trunk %s, the trunk is now %s "+ + "(%s merged)", m.ID, shortOf(judged, 12), shortOf(commit, 12), merged.ID) + g.Check.OutOfDateAt = now + g.Updated = now + h.keepGroup(g) + h.Logf("[mesh-delivery] group %s: %s", g.ID, g.Check.OutOfDate) + break + } + } } // closeGroupOf closes a delivery's group once it reached the trunk: a delivering group takes no new member. @@ -540,7 +634,13 @@ type CheckedEvent struct { Owner string `json:"owner"` Repo string `json:"repo"` Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` } `json:"members,omitempty"` + // Trunk is the trunk commit the head was merged into and judged as, Beside the other repositories' commits the + // check read, and Took how long it ran (novox/hq ADR 0323 §3). + Trunk string `json:"trunk,omitempty"` + Beside map[string]string `json:"beside,omitempty"` + Took string `json:"took,omitempty"` } // Checked takes a verdict: a head's own, or a group's composed check. @@ -555,6 +655,18 @@ func (h *Holder) Checked(c CheckedEvent) { return } g.Check.Verdict, g.Check.Summary = c.Verdict, c.Summary + // Each member's head and the trunk commit it was merged into (novox/hq ADR 0323 §6). + for _, m := range c.Members { + for id := range g.Check.Commits { + if d := h.deliveries[id]; d != nil && strings.EqualFold(d.Repository, m.Owner+"/"+m.Repo) && + sameCommit(d.Commit, m.Commit) && m.Trunk != "" { + if g.Check.Trunks == nil { + g.Check.Trunks = map[string]string{} + } + g.Check.Trunks[id] = m.Trunk + } + } + } g.Updated = now h.keepGroup(g) h.Logf("[mesh-delivery] group %s composed: %s — %s", g.ID, c.Verdict, c.Summary) @@ -575,11 +687,23 @@ func (h *Holder) Checked(c CheckedEvent) { } h.moved(d, []Transition{t}) } + if d.State == Rejected && c.Trunk != "" && d.Check != nil && !sameCommit(d.Check.Trunk, c.Trunk) { + // Judged again against another trunk commit — the merge tool's ask of a verdict out of date, or a check asked + // elsewhere (novox/hq ADR 0323 §5, §7): taken as the recheck it is. + t, err := Apply(d, EvRecheck, true, Facts{Now: now, By: byOwner, Why: fmt.Sprintf("judged again against trunk %s "+ + "(it was rejected against trunk %s)", shortOf(c.Trunk, 12), shortOf(d.Check.Trunk, 12))}) + if err != nil { + h.refuse(err) + return + } + h.moved(d, []Transition{t}) + } if d.State != Proposed { h.Logf("[mesh-delivery] %s is %s: a verdict for it now is history (%s)", d.ID, d.State, c.Verdict) return } - v := &Verdict{ID: c.ID, On: c.On, At: now, Gate: c.Verdict, Summary: c.Summary, Report: c.Report} + v := &Verdict{ID: c.ID, On: c.On, At: now, Gate: c.Verdict, Summary: c.Summary, Report: c.Report, + Head: shortOf(c.Commit, 12), Trunk: c.Trunk, Beside: c.Beside, Took: c.Took} if c.Gate != nil { v.Gate, v.Summary = c.Gate.Verdict, c.Gate.Summary v.Modules, v.Dependents = c.Gate.Modules, c.Gate.Dependents @@ -590,10 +714,14 @@ func (h *Holder) Checked(c CheckedEvent) { if v.Gate == "" { v.Gate = "error" } - d.Check = v + d.Check, d.AskedAgain = v, nil if c.Plan != nil { d.Plan = c.Plan } + if why := d.outOfDate(); why != "" { + // Kept and shown, and it does not count: the delivery stays proposed (ADR 0323 §7). + h.Logf("[mesh-delivery] %s: its verdict %s is %s", d.ID, c.ID, why) + } h.settleAndKeep(d) } @@ -1442,14 +1570,19 @@ func (h *Holder) planGroup(g *Group, members []*Delivery, now time.Time) []func( }) return asks } - // The composed check, once every member is ready and the order holds. + // The composed check, once every member is ready and the order holds. Asked again for new heads, even once the + // group is closed: a member rebased or pushed is a new head, and the group waits for its composed check rather + // than being held for a person (novox/hq ADR 0323 §6, issue 406). One out of date is asked again once a member + // was judged again after it went out of date: a recheck, or the merge tool's ask. allReady := true for _, d := range members { - if d.State != Ready { + // In a closed group a member on the trunk counts — merged and on its way, or already delivered (reached). + if d.State != Ready && !(g.Closed && d.MergedAs != "" && (!d.State.Final() || d.State.Reached())) { allReady = false } } - if allReady && len(g.Cycle) == 0 && !g.Closed && (g.Check == nil || !sameMembers(g.Check.Commits, commits)) { + again := g.Check != nil && (!sameMembers(g.Check.Commits, commits) || g.Check.OutOfDate != "" && rejudgedSince(members, g.Check.OutOfDateAt)) + if allReady && len(g.Cycle) == 0 && (g.Check == nil && !g.Closed || again) { ms := h.inOrder(g, members) id := g.ID asks = append(asks, func() { @@ -1571,7 +1704,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa } // Composed for these very heads: a verdict on another set of heads is no verdict on these. composed := g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") && - sameMembers(g.Check.Commits, commitsOf(members)) + sameMembers(g.Check.Commits, commitsOf(members)) && g.Check.OutOfDate == "" // A member's turn registered, the members after it that the same walk carries take theirs too: one walk is let // go once, for all of them (ADR 0276 decision 5). A member on another walk, or not at its turn, waits. var asks []func() @@ -1657,9 +1790,11 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa } return asks case d.State == Published: - // Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316). - if !composed && d.HeldWhy == "" && g.Check != nil && g.Check.Verdict == "" && - sameMembers(g.Check.Commits, commitsOf(members)) { + // Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316). Nor is a + // composed check of other heads — a member rebased or pushed — or one out of date: it runs again for these + // heads, and the member waits for it (novox/hq ADR 0323 §6, issue 406). + if !composed && d.HeldWhy == "" && g.Check != nil && (g.Check.Verdict == "" || g.Check.OutOfDate != "" || + !sameMembers(g.Check.Commits, commitsOf(members))) { return asks } if !composed && d.HeldWhy == "" { @@ -2180,12 +2315,23 @@ func GroupState(g *Group, members []*Delivery) (string, string) { return "delivered", "every member is delivered" case g.Closed && n > 0 && ended == n: return "stopped", supersededWhy + strings.Join(uncovered, ", ") + case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" || + !sameMembers(g.Check.Commits, commitsOf(members))): + // A member rebased or pushed once the group was delivering, or its composed check out of date: it is composed + // again for these heads, and the group is checking until then, never held (novox/hq ADR 0323 §6, issue 406). + if g.Check.OutOfDate != "" { + return "checking", g.Check.OutOfDate + } + return "checking", "composed again for its members' new heads" case g.Closed: return "delivering", fmt.Sprintf("%d of %d delivered", reached, n) case len(g.Cycle) > 0: return "rejected", cycleSaid(g) case count[Rejected] > 0: return "rejected", "a member's own check failed" + case g.Check != nil && g.Check.OutOfDate != "": + // Out of date counts as outstanding (ADR 0239 §18, novox/hq ADR 0323 §7): checking, saying why. + return "checking", g.Check.OutOfDate case g.Check != nil && g.Check.Verdict != "" && g.Check.Verdict != "pass" && g.Check.Verdict != "warning": return "rejected", "composed together: " + g.Check.Summary case count[Ready] == n && g.Check != nil && sameMembers(g.Check.Commits, commitsOf(members)) && @@ -2251,3 +2397,13 @@ func cycleSaid(g *Group) string { } return why } + +// rejudgedSince is whether a member was judged again after a moment: a verdict of its own heard since. +func rejudgedSince(members []*Delivery, at time.Time) bool { + for _, d := range members { + if d.Check != nil && d.Check.At.After(at) { + return true + } + } + return false +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go new file mode 100644 index 00000000..1b6e5352 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go @@ -0,0 +1,463 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// **A verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341). The check judges the head +// merged into a named trunk commit; when another merge moves that trunk, a ready delivery goes back to proposed, +// a rejected one stays rejected, and nothing is asked. A verdict for an old trunk is kept and shown and does not +// count, so the table reaches a fixed point. A group's composed check goes out of date on a merge that is not one of +// its members, and runs again for new heads instead of holding the group. + +const ( + trunk0 = "a2c2c35000000000000000000000000000000000" + merge1 = "d28ca95000000000000000000000000000000000" +) + +// judged is a verdict of a head merged into a trunk commit. +func judged(repo string, number int, sha, gate, trunk string) CheckedEvent { + c := verdict(repo, number, sha, gate) + c.Trunk = shortOf(trunk, 12) + c.ID = "check-of-" + sha + "-at-" + shortOf(trunk, 8) + c.Took = "4m12s" + c.Beside = map[string]string{"mesh-catalog": "0123456789ab"} + plan := *c.Plan + plan.Head, plan.Trunk = sha, c.Trunk + c.Plan = &plan + return c +} + +func TestATrunkMoveSendsAReadyDeliveryBackToProposed(t *testing.T) { + const host = "novox/mesh-host" + const h64, h66, h67 = "6464646464646464", "dfb1415dfb1415df", "6767676767676767" + w := newWorld(t) + for _, p := range []PullEvent{pr(host, 64, h64, "a"), pr(host, 66, h66, "b"), pr(host, 67, h67, "c")} { + w.h.PullUpdated(p) + } + w.h.Checked(judged(host, 64, h64, "pass", trunk0)) + w.h.Checked(judged(host, 66, h66, "pass", trunk0)) + w.h.Checked(judged(host, 67, h67, "fail", trunk0)) + id66, id67 := IDOf(host, h66), IDOf(host, h67) + if s := w.state(id66); s != Ready { + t.Fatalf("#66 checked green against the trunk is %s", s) + } + asked := len(w.ctl.checks) + + // #64 merges: the trunk moves to its merge commit. + w.h.PullMerged(merged(host, 64, h64, merge1)) + w.settleAll() + d := w.delivery(id66) + if d.State != Proposed { + t.Fatalf("#66's verdict names trunk %s, the trunk is now %s, and it is %s", trunk0[:12], merge1[:12], d.State) + } + last := d.Transitions[len(d.Transitions)-1] + if last.Event != EvTrunkMoved || !strings.Contains(last.Why, trunk0[:12]) || !strings.Contains(last.Why, merge1[:12]) { + t.Fatalf("its transition was %s, %q: not trunk moved with both commits", last.Event, last.Why) + } + if d.Check == nil || d.Check.Trunk != trunk0[:12] { + t.Fatalf("its out-of-date verdict was not kept to be shown: %+v", d.Check) + } + if s := w.state(id67); s != Rejected { + t.Fatalf("a rejected delivery went to %s on the trunk's move", s) + } + if len(w.ctl.checks) != asked { + t.Fatalf("the trunk's move asked a check: %v", w.ctl.checks[asked:]) + } + + // Out of date and not asked: no bound, so stalled does not list it, however long it waits. + w.later(5 * time.Hour) + w.settleAll() + for _, l := range w.h.Stalled() { + if l.ID == id66 { + t.Fatalf("stalled lists a proposed delivery whose check is out of date and not asked: %+v", l) + } + } + if len(w.ctl.checks) != asked { + t.Fatalf("an out-of-date check was asked by this owner: %v", w.ctl.checks[asked:]) + } + + // A late verdict for the old trunk is kept, does not count, and moves nothing: the table's fixed point. + before := len(w.delivery(id66).Transitions) + late := judged(host, 66, h66, "pass", trunk0) + late.ID = "check-late" + w.h.Checked(late) + w.settleAll() + d = w.delivery(id66) + if d.State != Proposed || len(d.Transitions) != before || d.Check == nil || d.Check.ID != "check-late" { + t.Fatalf("a late verdict for an old trunk left it %s after %d transition(s), verdict %+v", d.State, + len(d.Transitions)-before, d.Check) + } + if ts := settle(d, Facts{Now: w.now}); len(ts) != 0 { + t.Fatalf("the table took %d more transition(s) on a verdict that does not count", len(ts)) + } + + // The check asked again judges the head merged into the trunk's head: it counts. + w.h.Checked(judged(host, 66, h66, "pass", merge1)) + w.settleAll() + if s := w.state(id66); s != Ready { + t.Fatalf("a verdict judged against the trunk's head left it %s", s) + } +} + +// A rejected delivery judged again against another trunk — the merge tool's ask — is taken as its recheck. +func TestARejectedDeliveryJudgedAgainstANewTrunkIsRecheckedByTheVerdict(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "fail", trunk0)) + id := IDOf(repo, head) + if s := w.state(id); s != Rejected { + t.Fatalf("it is %s", s) + } + w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1)) + w.h.Checked(judged(repo, 9, head, "pass", merge1)) + w.settleAll() + d := w.delivery(id) + if d.State != Ready || d.Check.Trunk != merge1[:12] { + t.Fatalf("judged green against the trunk's head it is %s with %+v", d.State, d.Check) + } +} + +// **A verdict names the trunk commit it judged** in `checks` and `show`, and the kept plan records it (ADR 0323 §3). +func TestChecksAndShowNameTheHeadAndTheTrunkCommit(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "pass", trunk0)) + id := IDOf(repo, head) + + a, err := w.h.Checks(repo, 0, head) + if err != nil { + t.Fatal(err) + } + want := "head " + head[:12] + " merged into trunk " + trunk0[:12] + if a.Judged != want || a.Verdict == nil || a.Verdict.Head != head[:12] || a.Verdict.Trunk != trunk0[:12] { + t.Fatalf("checks says %q, verdict %+v; want %q", a.Judged, a.Verdict, want) + } + if !strings.Contains(a.Says, want) { + t.Fatalf("checks' words do not say what was judged: %s", a.Says) + } + if a.Verdict.Beside["mesh-catalog"] != "0123456789ab" || a.Verdict.Took != "4m12s" { + t.Fatalf("checks does not name the other repositories' commits or how long it took: %+v", a.Verdict) + } + shown, err := w.h.Show(id) + if err != nil { + t.Fatal(err) + } + d := shown.(map[string]any)["delivery"].(*Delivery) + if d.Check.Head != head[:12] || d.Check.Trunk != trunk0[:12] { + t.Fatalf("show gives the verdict %+v", d.Check) + } + if d.Plan == nil || d.Plan.Trunk != trunk0[:12] || d.Plan.Head != head { + t.Fatalf("the kept delivery plan records %+v", d.Plan) + } + + w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1)) + if a, _ = w.h.Checks(repo, 0, head); !strings.Contains(a.OutOfDate, merge1[:12]) { + t.Fatalf("checks does not say the verdict is out of date: %+v", a) + } +} + +// **A delivery group's composed check follows the same rule** (ADR 0323 §6, §7): a merge that is not one of its +// members into a member's trunk puts it out of date and the group reads checking; its own members merging in its +// order do not; and it runs again once a member is judged again. +func TestAGroupsComposedCheckGoesOutOfDateOnAMergeThatIsNotItsMember(t *testing.T) { + const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog" + const hc, hk = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1" + idc, idk := IDOf(ctl, hc), IDOf(cat, hk) + start := func(t *testing.T) *world { + w := newWorld(t) + w.h.PullUpdated(pr(ctl, 1, hc, "feat/g")) + w.h.PullUpdated(pr(cat, 2, hk, "feat/g")) + w.h.Checked(judged(ctl, 1, hc, "pass", trunk0)) + w.h.Checked(judged(cat, 2, hk, "pass", trunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + if g == nil || g.Check == nil { + t.Fatalf("no composed check was asked: %+v", g) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: hc, Trunk: trunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: hk, Trunk: trunk0[:12]}}}) + w.settleAll() + if state, _ := GroupState(g, w.h.membersOf(g)); state != "ready" { + t.Fatalf("the group composed green is %s", state) + } + if g.Check.Trunks[idc] != trunk0[:12] { + t.Fatalf("the composed check names its members' trunks %v", g.Check.Trunks) + } + return w + } + + t.Run("a merge that is not a member", func(t *testing.T) { + w := start(t) + asked := len(w.ctl.checks) + w.h.PullMerged(merged(ctl, 5, "5555555555555555", merge1)) + w.settleAll() + g := w.h.groups["feat/g"] + state, why := GroupState(g, w.h.membersOf(g)) + if state != "checking" || g.Check.OutOfDate == "" { + t.Fatalf("the group is %s (%s) after a merge that is not its member moved a member's trunk", state, why) + } + for _, head := range []string{ctl + "@" + hc, cat + "@" + hk} { + s := w.forge.statuses[head+" "+ctxGroup] + if !strings.HasPrefix(s, "pending ") || !strings.Contains(s, "out of date") { + t.Fatalf("mesh/delivery-group on %s says %q, not pending with the reason", head, s) + } + } + if len(w.ctl.checks) != asked { + t.Fatalf("the trunk's move asked a check: %v", w.ctl.checks[asked:]) + } + // The controller's member judged again against the trunk's head: the composed check runs again. + w.later(time.Minute) + w.h.Checked(judged(ctl, 1, hc, "pass", merge1)) + w.settleAll() + if len(w.ctl.checks) == asked || !strings.HasPrefix(w.ctl.checks[len(w.ctl.checks)-1], "feat/g:") { + t.Fatalf("the composed check was not asked again once a member was judged again: %v", w.ctl.checks) + } + }) + + t.Run("its own member merging", func(t *testing.T) { + w := start(t) + mc := merged(ctl, 1, hc, merge1) + mc.Head = "feat/g" + w.h.PullMerged(mc) + w.settleAll() + g := w.h.groups["feat/g"] + if g.Check.OutOfDate != "" { + t.Fatalf("the group's own member merging put its composed check out of date: %s", g.Check.OutOfDate) + } + if s := w.state(idk); s != Ready { + t.Fatalf("the catalogue's member, in another repository, is %s", s) + } + }) +} + +// **After a member is rebased or pushed, the composed check runs again for the new heads** and the group reads +// checking, not held (ADR 0323 §6, issue 406's second symptom): its first member merged, closing the group, and then +// its second was rebased just before its own merge. +func TestARebasedMemberIsComposedAgainNotHeld(t *testing.T) { + const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog" + const hc, hk, hc2 = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1", "c2c2c2c2c2c2c2c2" + idc, idk, idc2 := IDOf(ctl, hc), IDOf(cat, hk), IDOf(ctl, hc2) + w := newWorld(t) + w.h.PullUpdated(pr(ctl, 1, hc, "feat/g")) + w.h.PullUpdated(pr(cat, 2, hk, "feat/g")) + w.h.Checked(judged(ctl, 1, hc, "pass", trunk0)) + w.h.Checked(judged(cat, 2, hk, "pass", trunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + if len(g.Order) != 2 || g.Order[0] != idk { + t.Fatalf("the group's order is %v: the catalogue's member first is what this replays", g.Order) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}) + w.settleAll() + + // The catalogue's member merges first: the group is closed. + mk := merged(cat, 2, hk, merge1) + mk.Head = "feat/g" + w.h.PullMerged(mk) + w.h.WalkMoved(w.ctl.put(aWalk("plan-cat", cat, merge1, true))) + w.settleAll() + if !g.Closed { + t.Fatal("the group is not closed once a member merged") + } + + // The controller's member is rebased just before its merge: a new head, with its own check. + w.h.PullUpdated(pr(ctl, 1, hc2, "feat/g")) + w.settleAll() + if d := w.delivery(idc2); d.Group != "feat/g" { + t.Fatalf("the rebased head did not take its place in its closed group: %q", d.Group) + } + if containsString(g.Members, idc) || !containsString(g.Members, idc2) || len(g.Order) != 2 || g.Order[1] != idc2 { + t.Fatalf("the group's members are %v in the order %v", g.Members, g.Order) + } + if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" { + t.Fatalf("the group reads %s (%s), not checking", state, why) + } + w.h.Checked(judged(ctl, 1, hc2, "pass", trunk0)) + w.settleAll() + asked := w.ctl.checks[len(w.ctl.checks)-1] + if !strings.HasPrefix(asked, "feat/g:") || !strings.Contains(asked, idc2) || !strings.Contains(asked, idk) { + t.Fatalf("the composed check was not asked again for the new heads: %v", w.ctl.checks) + } + if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" { + t.Fatalf("the group reads %s (%s), not checking", state, why) + } + for _, id := range []string{idk, idc2} { + if d := w.delivery(id); d.State == Held { + t.Fatalf("%s is held for a person: %s", id, d.HeldWhy) + } + } + // Its verdict for the new heads: the rebased member merges and takes its turn. + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"}) + mc := merged(ctl, 1, hc2, "e0e0e0e0e0e0e0e0") + mc.Head = "feat/g" + w.h.PullMerged(mc) + w.h.WalkMoved(w.ctl.put(aWalk("plan-ctl", ctl, "e0e0e0e0e0e0e0e0", true))) + w.settleAll() + if d := w.delivery(idc2); d.State == Held { + t.Fatalf("the rebased member, composed green for the new heads, is held: %s", d.HeldWhy) + } +} + +// A head whose merge check the forge's holder put out of date is not a check asked and not answered: `stalled` does +// not list it, however long it waits — rejected, or with no delivery here — until its check is asked (ADR 0323 §7). +func TestAnOutOfDateCheckOnTheForgeIsNotSaidUnanswered(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b9c0d" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "fail", trunk0)) + words := outOfDateWords(trunk0, merge1) + pending := func(description, at string) map[string]HeadCheck { + return map[string]HeadCheck{ctxGate: {State: "pending", Description: description, UpdatedAt: at}, + ctxRepo: {State: "pending", Description: description, UpdatedAt: at}} + } + w.forge.open = []OpenPull{{Owner: "novox", Repo: "mesh-lab", Number: 9, Base: "main", HeadSHA: head, + Required: []string{ctxGate, ctxRepo}, Statuses: []string{ctxGate, ctxRepo}, Checks: pending(words, "2026-10-06T12:01:00Z")}} + w.h.SeeOpenPulls() + w.later(3 * time.Hour) + w.h.SeeOpenPulls() + for _, l := range w.h.Stalled() { + if strings.HasPrefix(l.ID, repo) { + t.Fatalf("an out-of-date check nobody asked is said stalled: %+v", l) + } + } + // Asked by the merge tool: pending anew, and its bound runs from then. + w.forge.open[0].Checks = pending("waiting in the build queue since 15:01: asked by the merge tool", "2026-10-06T15:01:00Z") + w.h.SeeOpenPulls() + w.later(2 * time.Hour) + w.h.SeeOpenPulls() + said := false + for _, l := range w.h.Stalled() { + said = said || l.State == Unanswered + } + if !said { + t.Fatal("a check the merge tool asked and that never answered is not said") + } +} + +// The merge tool's ask announces an out-of-date head again: its one-hour bound runs from then, and a check that never +// answers is said stalled (ADR 0323 §7). +func TestAnOutOfDateDeliveryAskedAgainIsBoundFromTheAsk(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "pass", trunk0)) + w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1)) + id := IDOf(repo, head) + w.later(3 * time.Hour) + w.h.PullUpdated(pr(repo, 9, head, "x")) // the merge tool asked its check + w.later(30 * time.Minute) + for _, l := range w.h.Stalled() { + if l.ID == id { + t.Fatalf("listed half an hour after its check was asked again: %+v", l) + } + } + w.later(time.Hour) + said := false + for _, l := range w.h.Stalled() { + if l.ID == id { + said = true + if !strings.Contains(l.Says, "asked again") || !strings.Contains(l.H2, "operator") { + t.Fatalf("its stalled line says %+v", l) + } + } + } + if !said { + t.Fatal("a check asked again that never answered is not said") + } + // Its verdict judged against the trunk's head: ready, and nothing bounds it as proposed. + w.h.Checked(judged(repo, 9, head, "pass", merge1)) + if d := w.delivery(id); d.State != Ready || d.AskedAgain != nil { + t.Fatalf("judged again it is %s, asked again %v", d.State, d.AskedAgain) + } +} + +// The same, with the first member already delivered when the second is rebased: a delivered member counts for the +// composed check of the new heads, which is asked, and the rebased member is never held (review of ADR 0323 §6). +func TestARebasedMemberAfterAnotherWasDeliveredIsComposedAgain(t *testing.T) { + const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog" + const hc, hk, hc2 = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1", "c2c2c2c2c2c2c2c2" + idk, idc2 := IDOf(cat, hk), IDOf(ctl, hc2) + w := newWorld(t) + w.h.PullUpdated(pr(ctl, 1, hc, "feat/g")) + w.h.PullUpdated(pr(cat, 2, hk, "feat/g")) + w.h.Checked(judged(ctl, 1, hc, "pass", trunk0)) + w.h.Checked(judged(cat, 2, hk, "pass", trunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}) + w.settleAll() + + // The catalogue's member merges and its walk is done: delivered. + mk := merged(cat, 2, hk, merge1) + mk.Head = "feat/g" + w.h.PullMerged(mk) + walk := w.ctl.put(aWalk("plan-cat", cat, merge1, true)) + w.h.WalkMoved(walk) + w.settleAll() + walk = w.ctl.walk("plan-cat") + walk.State = walkDone + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(idk); s != Delivered { + t.Fatalf("the catalogue's member is %s, not delivered", s) + } + + // The controller's member is rebased, and judged. + w.h.PullUpdated(pr(ctl, 1, hc2, "feat/g")) + w.h.Checked(judged(ctl, 1, hc2, "pass", trunk0)) + w.settleAll() + asked := w.ctl.checks[len(w.ctl.checks)-1] + if !strings.HasPrefix(asked, "feat/g:") || !strings.Contains(asked, idc2) { + t.Fatalf("the composed check was not asked again for the new heads: %v", w.ctl.checks) + } + if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" { + t.Fatalf("the group reads %s (%s), not checking", state, why) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"}) + mc := merged(ctl, 1, hc2, "e0e0e0e0e0e0e0e0") + mc.Head = "feat/g" + w.h.PullMerged(mc) + w.h.WalkMoved(w.ctl.put(aWalk("plan-ctl", ctl, "e0e0e0e0e0e0e0e0", true))) + w.settleAll() + if d := w.delivery(idc2); d.State == Held { + t.Fatalf("the rebased member is held: %s", d.HeldWhy) + } +} + +// A check this owner asks — a recheck — sends the base head the forge last announced the head with, so the +// controller's verdict for a head no build judges names its trunk commit (ADR 0323 §3, review of mesh-controller#271). +func TestARecheckSendsTheAnnouncedBaseHead(t *testing.T) { + const repo = "novox/mesh-tools-go" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + p := pr(repo, 3, head, "x") + p.BaseSHA = trunk0 + w.h.PullUpdated(p) + id := IDOf(repo, head) + if m := MemberOf(w.delivery(id)); m.BaseSHA != trunk0 { + t.Fatalf("the member sends base head %q", m.BaseSHA) + } + p.BaseSHA = merge1 // announced again, by the merge tool + w.h.PullUpdated(p) + if m := MemberOf(w.delivery(id)); m.BaseSHA != merge1 { + t.Fatalf("announced again, the member sends base head %q", m.BaseSHA) + } + if raw := string(mustJSON(MemberOf(w.delivery(id)))); !strings.Contains(raw, `"base_sha":"`+merge1+`"`) { + t.Fatalf("the member as the controller reads it: %s", raw) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index c22b22a4..37f0957b 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -49,6 +49,17 @@ type Delivery struct { // CheckAsk is this owner's own ask of its check, when the forge's announcement did not bring one: a head // adopted while it already waited, or one whose announcement the controller missed. CheckAsk *CheckAsk `json:"check_ask,omitempty"` + // TrunkNow is its trunk's head as this owner last heard it move while it was open: the commit the last merge + // into its trunk landed as (novox/hq ADR 0323 §7). A verdict that names another trunk commit is out of date. + // Empty until a merge into its trunk is heard after it was made. + TrunkNow string `json:"trunk_now,omitempty"` + // BaseSHA is the base branch's head the forge last announced its head with (ADR 0323): sent with a check this + // owner asks, for a verdict no build judges to name its trunk commit. + BaseSHA string `json:"base_sha,omitempty"` + // AskedAgain is when its head was announced again while its verdict was out of date — the merge tool's ask of + // its check against the trunk's head (ADR 0323 §5) — and no verdict has come since: its one-hour bound runs from + // then. Nil while nothing is asked, and cleared by a verdict or another move of its trunk. + AskedAgain *time.Time `json:"asked_again,omitempty"` // The facts the table's guards read. NewerHead is the head that superseded it; ClosedUnmerged the pull // request closed; MergedAs the commit it landed on the trunk as; HeldWhy what holds it for a person. @@ -128,6 +139,38 @@ type Verdict struct { Repo string `json:"repo,omitempty"` RepoSaid string `json:"repo_summary,omitempty"` Report string `json:"report,omitempty"` + // Head and Trunk are what it judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, twelve + // characters each. Trunk is empty for a verdict of a build seat that judged the head alone. Beside are the + // commits of the other repositories the check read, by directory; Took how long it ran. + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Beside map[string]string `json:"beside,omitempty"` + Took string `json:"took,omitempty"` +} + +// Judged is what a verdict judged, in words: "head 1a2b3c4d5e6f merged into trunk 7a8b9c0d1e2f" (ADR 0323 §3); +// empty for a verdict that names no trunk commit. +func (v *Verdict) Judged() string { + if v == nil || v.Trunk == "" { + return "" + } + return fmt.Sprintf("head %s merged into trunk %s", shortOf(v.Head, 12), shortOf(v.Trunk, 12)) +} + +// outOfDate is why a delivery's verdict no longer counts: it names a trunk commit that is not its trunk's head +// as this owner heard it (novox/hq ADR 0323 §7). Empty when it counts — or names no trunk commit, as a verdict +// of a build seat from before the rule does, which counts as it did. +func (d *Delivery) outOfDate() string { + if d.Check == nil || d.Check.Trunk == "" || d.TrunkNow == "" || sameCommit(d.Check.Trunk, d.TrunkNow) { + return "" + } + return outOfDateWords(d.Check.Trunk, d.TrunkNow) +} + +// outOfDateWords is a verdict out of date, as the forge's holder says it on the pull request (ADR 0323 §4). +func outOfDateWords(judged, now string) string { + return fmt.Sprintf("out of date: judged against trunk %s, the trunk is now %s; judged again on a recheck or when "+ + "it is merged", shortOf(judged, 12), shortOf(now, 12)) } // CheckAsk is a head's own check, asked of the controller by this owner (or re-asked by healer H2). @@ -168,6 +211,8 @@ type DeliveryPlan struct { // Class is the class of the slowest-budgeted module it moves, as the planner says it: core, leaf, or none when // it moves no module (novox/hq ADR 0313 §2). Empty in a plan from a controller that did not say it. Class string `json:"class,omitempty"` + // Trunk is the trunk commit the head was merged into when its verdict judged it (novox/hq ADR 0323 §3). + Trunk string `json:"trunk,omitempty"` } // receiving are the machines the plan sends a module to, in the plan's order; none when the plan names no @@ -755,6 +800,13 @@ type GroupCheck struct { Commits map[string]string `json:"commits"` Verdict string `json:"verdict,omitempty"` Summary string `json:"summary,omitempty"` + // Trunks are the trunk commit each member's head was merged into, by member id (novox/hq ADR 0323 §6). + Trunks map[string]string `json:"trunks,omitempty"` + // OutOfDate is why its verdict no longer counts: a merge that is not one of the group's members moved a + // member's trunk, at OutOfDateAt. It counts as outstanding (ADR 0239 §18), and runs again once a member is + // judged again after it. + OutOfDate string `json:"out_of_date,omitempty"` + OutOfDateAt time.Time `json:"out_of_date_at"` } // Commits is each member's commit, by id. diff --git a/modules/mesh-delivery/cmd/mesh-delivery/ports.go b/modules/mesh-delivery/cmd/mesh-delivery/ports.go index 8675075e..c0185b1d 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/ports.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/ports.go @@ -31,13 +31,16 @@ type Member struct { ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` CloneURL string `json:"clone_url,omitempty"` After []string `json:"after,omitempty"` + // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit + // the controller's verdict names when no build judges the head. + BaseSHA string `json:"base_sha,omitempty"` } // MemberOf is a delivery as a member. func MemberOf(d *Delivery) Member { return Member{ID: d.ID, Repository: d.Repository, Base: d.Base, Head: d.Commit, Number: d.Number, Paths: d.Paths, PathsTruncated: d.PathsTruncated, Removed: d.Removed, ModuleDirs: d.ModuleDirs, ModuleDirsSaid: d.ModuleDirsSaid, - CloneURL: d.CloneURL, After: d.After} + CloneURL: d.CloneURL, After: d.After, BaseSHA: d.BaseSHA} } // Order is a group's order as the controller works it out: each pair with the rules it won over by diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index 5203ee17..04fa603f 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -85,6 +85,7 @@ const ( EvCovered Event = "covered" // a later merge into the same trunk delivered every module it moves EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed, or a walk now answers its merge EvRewalk Event = "rewalk" // the walk that carried a later commit for it failed, or was stopped for another merge (ADR 0299): it waits for the walk that walks it again + EvTrunkMoved Event = "trunk moved" // another merge moved its trunk, and its verdict names the trunk commit before (novox/hq ADR 0323) ) // Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act. @@ -167,9 +168,17 @@ var Table = []Row{ Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "" && (d.Walk != nil || d.MovesNothing()), "not on the trunk yet") }}, + // **A verdict counts only for the trunk's head** (novox/hq ADR 0323 §7): one naming a trunk commit that is no + // longer the trunk's head is kept and shown, and does not count, so the delivery stays proposed and the rows + // cannot loop with `trunk moved`; a recheck or the merge tool's ask judges it again. {From: []State{Proposed}, Event: EvChecked, To: Checked, - Guard: "the verdict names this commit", - Holds: func(d *Delivery, f Facts) error { return unless(d.Check != nil, "no verdict") }}, + Guard: "a verdict names this commit and the trunk's head as its trunk commit", + Holds: func(d *Delivery, f Facts) error { + if d.Check == nil { + return errors.New("no verdict") + } + return unless(d.outOfDate() == "", "its verdict is "+d.outOfDate()) + }}, {From: []State{Checked}, Event: EvAccepted, To: Ready, Guard: "the gate passed or warned, and the repository's own check did not fail", Holds: func(d *Delivery, f Facts) error { return unless(d.Check.Passes(), "the verdict does not pass") }}, @@ -199,6 +208,14 @@ var Table = []Row{ Says: func(d *Delivery) string { return "covered: a later merge that contains it was delivered: " + d.CoveredBy }}, + // **A ready delivery whose verdict went out of date goes back to proposed** (novox/hq ADR 0323 §7): another + // merge moved its trunk. A rejected one stays rejected, its status on the forge saying the verdict is out of date. + {From: []State{Ready}, Event: EvTrunkMoved, To: Proposed, + Guard: "its verdict names a trunk commit that is not its trunk's head", + Holds: func(d *Delivery, f Facts) error { + return unless(d.MergedAs == "" && d.outOfDate() != "", "not merged, and its verdict is not out of date") + }, + Says: func(d *Delivery) string { return d.outOfDate() }}, {From: []State{Proposed, Checked, Rejected}, Event: EvMergedUnchecked, To: Held, Guard: "merged without a passing check — a verdict known with it is taken first: only a person decides that it goes on", Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "", "not merged") }}, @@ -543,6 +560,9 @@ const checkGrace = 15 * time.Minute // runs from its start plus the grace before it is asked: an ask that cannot be made is still listed. func boundSince(d *Delivery) time.Time { if d.State == Proposed { + if d.AskedAgain != nil && d.outOfDate() != "" { + return *d.AskedAgain + } if d.CheckAsk != nil { return d.CheckAsk.At } @@ -562,6 +582,17 @@ var rewalkBound = Bound{For: unwalked, Says: "the walk that carried a later comm // boundOf is a delivery's bound: its state's, or none for a delivery held because no walk answers it. func boundOf(d *Delivery) (Bound, bool) { + if d.State == Proposed && d.outOfDate() != "" { + // Its check is out of date and not asked: nothing is running for it, and it costs nothing (ADR 0323 §7). + // Its bound starts when its check is asked: a recheck puts the verdict away; the merge tool's ask announces the + // head again, and its bound runs from then. + if d.AskedAgain == nil { + return Bound{}, false + } + b := Bounds[Proposed] + b.H2, b.Says = nil, "its verdict is "+d.outOfDate()+"; its check was asked again and not answered: the operator's" + return b, true + } if d.State == Held && d.Unwalked && d.Walk == nil { return heldUnwalked, true } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go index 75f5fecc..c1f902cf 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -82,6 +82,9 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { case EvRewalk: d.MergedAs, d.AnsweredBy = "aaaa1111", "bbbb2222" d.Walk = &WalkSeen{ID: "plan-2", State: walkFailed} + case EvTrunkMoved: + d.Check = &Verdict{Gate: "pass", Head: "c0ffee000000", Trunk: "d28ca95bbbbb"} + d.TrunkNow = "690966dccccccccc" case EvLifted: d.MergedAs, d.Lifted = "aaaa1111", "its group's composed check passed" d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} @@ -129,7 +132,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead, EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop, - EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk} + EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk, EvTrunkMoved} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go b/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go index 0db11ed5..2947b823 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go @@ -44,6 +44,10 @@ const Unannounced State = "unannounced" // own for the proposed state's bound: its check was asked and not answered. const Unanswered State = "unanswered" +// outOfDatePrefix begins the words the forge's holder puts a merge check back to pending with when its trunk moved +// (novox/hq ADR 0323 §4): the gitea module's outOfDateWords. +const outOfDatePrefix = "out of date:" + // UncheckedAfter is how long a head may go without a merge check before it is said: the announcer passes every half // minute and the check is asked at once, so ten minutes is a fault, never a queue. const UncheckedAfter = 10 * time.Minute @@ -119,6 +123,9 @@ func (p OpenPull) unanswered() (waiting []string, pending map[string]string, sta // since nothing here says whether the one it has answered. case !ok || got.State == "": waiting = append(waiting, c+" never set") + case got.State == "pending" && strings.HasPrefix(got.Description, outOfDatePrefix): + // Out of date because its trunk moved, and not asked (novox/hq ADR 0323 §4, §7): nothing runs for it and + // it costs nothing, so it is not a check asked and not answered. Asked again, it is pending anew. case got.State == "pending": pending[c] = utcOf(got.UpdatedAt) waiting = append(waiting, c+" pending since "+orUnknown(pending[c])) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index 07f5642d..bcaa8e57 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -229,6 +229,8 @@ func (h *Holder) Stalled() []StalledLine { b, _ := boundOf(d) h2 := "none: the state is the operator's" switch { + case d.State == Proposed && d.AskedAgain != nil && d.outOfDate() != "": + h2 = "none: its verdict is out of date and its check, asked again, has not answered — the operator's" case d.State == Proposed && d.CheckAsk == nil: h2 = "none: its check could not be asked yet — the controller does not answer delivery-check" case d.State == Proposed && d.CheckAsk.Reasked > 0: diff --git a/modules/mesh-reads/README.md b/modules/mesh-reads/README.md new file mode 100644 index 00000000..2981bdc3 --- /dev/null +++ b/modules/mesh-reads/README.md @@ -0,0 +1,63 @@ +# mesh-reads + +A node's files and accounts, read through the mesh (novox/hq ADR 0310, extending ADR 0268 §5; hq to-be 41 +WP8; issue 418). It holds the node seat `node-reads` on every node, the control-node included, so reading a +file, searching one, listing a directory, the owner and mode of each component of a path and looking up an +account or a group never go through the login shell's `execute`, and an approval is asked for an act, never +for a read. + +## Who it reads as + +The account `mesh-reads`, which this module makes on every node (`runs-as`, ADR 0259 §8): no login shell, a +home of its own under `/var/lib`, and no group beside its own. The tool runner serves it in a runtime of its +own, on its own bus credential, never the machine's runtime, which runs as the operator's account. So the +node's file modes bound what it sees: a file only the operator, root or a module's own account may read is +refused by its mode, and the answer says so. It refuses every read when it finds itself running as root. + +## What it refuses by name + +Before any read, whatever the file modes would allow (ADR 0310 §5): + +- **The places the mesh keeps a secret on this node**, which the controller composes from the manifests + and gives it as `MESH_SECRET_PLACES`: the directory of every module's own secret (its bus credential + among them), every credential file and grant directory, the vault's directories, and where the + operator-sealed secrets are kept. Without that list the module does not start. +- **The known credential files of a machine's programs** (`refuse.go`): an account's `.ssh`, `.gnupg`, + `.aws`, `.kube`; the agent's `.credentials.json`, `.netrc`, `.pgpass`, `.git-credentials`; private key + names (`id_ed25519`, `*.key`, `ssh_host_*_key`); `/etc/shadow`; `/etc/ssl/private`, `/etc/wireguard` + and the network manager's connections. + +- **Histories and profiles**: shell and program histories (`.zsh_history`, `.bash_history`, `.psql_history`, + `.viminfo` …), browser and mail profiles (`.mozilla`, `.config/google-chrome`, `.config/chromium` …), a + process's `environ`, `mem`, `fd`, `map_files`, `root` and `cwd` under `/proc`, and `/etc/rancher` and + `/etc/kubernetes/admin.conf`. + +**What was opened is judged, not the name.** The type is checked before the open, so a device or a fifo is +never opened; the file is opened non-blocking (in a search without following a link), checked again on its +descriptor, and the path the kernel gives for that descriptor is judged by name like any path asked. A link +flipped between the check and the open therefore reaches nothing a name refuses, and a deleted file is refused. +**Limit:** a hard link to a secret file under another name is not seen by the name rule; the file's own mode, +and the kernel's protected hard links, are what bound it. `stat` and `readlink` say a secret place's owner +and mode, and mark it, because those are not its content; `list`, `read` and `search` refuse it. + +## The content rule + +Every answer passes the content rule's secret class, from `mesh-sdk`'s `outward` package (ADR 0310 §4): a +line of a file or a matching line shaped like a secret is withheld and its number said, as is a link target +or an account's words. Every line of a `BEGIN … END` block — a private key's or a certificate's — is withheld whole, because one +base64 line of a key can pass the rule alone. A path and an entry's name are not judged: they are what the caller named or what the +file system calls a thing, and the rule would take a temporary directory's or a container layer's name for a +token. + +## Its verbs and their bounds + +| verb | replaces | bounds | +|---|---|---| +| `stat` | `stat`, `ls -ld`, `namei -l`, `findmnt -T` | 200 paths, 64 components each; links followed only with `follow` | +| `readlink` | `readlink`, `readlink -f`, `realpath` | one path | +| `list` | `ls`, `ls -la` | 2000 entries, cut and said | +| `read` | `cat`, `head`, `tail`, `less` | 64 KiB, or up to 256 KiB when asked; cut at a line and said; text only | +| `search` | `grep`, `grep -r`, `grep -F` | a fixed string; 100 matches (500 asked), 4 deep (8 asked), 5000 files, 4 MiB a file, 20 seconds (checked inside a file too); the named path's file system only, the others named; the bound met is said | +| `account` | `getent passwd`, `getent group`, `id`, `groups` | 2000 accounts and groups; never a password field | + +No answer is larger than 1 MiB. Nothing here writes a file. diff --git a/modules/mesh-reads/cmd/mesh-reads/main.go b/modules/mesh-reads/cmd/mesh-reads/main.go new file mode 100644 index 00000000..cb20f6f5 --- /dev/null +++ b/modules/mesh-reads/cmd/mesh-reads/main.go @@ -0,0 +1,204 @@ +// mesh-reads (novox/hq ADR 0310, to-be 41 WP8): the holder of node-reads. A node's files and accounts read +// through the mesh — the owner and mode of each component of a path, a link, a directory, a text file, a search +// by fixed string, an account or a group — so these reads never go through the login shell's execute. +// +// It runs as an account of its own, mesh-reads, served by the tool runner in a runtime of its own (ADR 0259 +// §8): in no group that holds a secret, with no login shell and no home of the operator's, so the node's file +// modes bound what it sees. On top, it refuses by name the places a secret is kept (refuse.go) and withholds +// any piece of an answer shaped like a secret (mesh-sdk's outward). Read-only: nothing here writes a file. +// +// stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Seat is the role this module holds. +const Seat = "node-reads" + +// PlacesWord is where the controller names this node's secret places (novox/hq ADR 0310 §5). +const PlacesWord = "MESH_SECRET_PLACES" + +func main() { + places, err := placesFrom(os.Getenv(PlacesWord)) + if err != nil { + fmt.Fprintf(os.Stderr, "[mesh-reads] %v\n", err) + os.Exit(1) + } + if err := stdio.Serve("", tools(newReader(places))); err != nil { + fmt.Fprintf(os.Stderr, "[mesh-reads] %v\n", err) + os.Exit(1) + } +} + +// placesFrom reads the controller's list. Absent is refused: without it, a module's own-secret directory +// would be read wherever the account's modes allow, and a refusal the decision promises would be silent. +func placesFrom(raw string) ([]string, error) { + if strings.TrimSpace(raw) == "" { + return nil, fmt.Errorf("%s is not given, so this node's secret places are unknown and nothing would be "+ + "refused by them (novox/hq ADR 0310 §5): a controller that knows node-reads composes it", PlacesWord) + } + var places []string + if err := json.Unmarshal([]byte(raw), &places); err != nil { + return nil, fmt.Errorf("%s is not a list of paths: %v", PlacesWord, err) + } + return places, nil +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func arg(a map[string]any, k string) string { + switch v := a[k].(type) { + case string: + return strings.TrimSpace(v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case bool: + return strconv.FormatBool(v) + } + return "" +} + +func number(a map[string]any, k string) (int, error) { + s := arg(a, k) + if s == "" { + return 0, nil + } + n, err := strconv.Atoi(s) + if err != nil { + return 0, fmt.Errorf("%s is a whole number, and %q was given", k, s) + } + return n, nil +} + +// pathsArg is the paths given, one per line — or, from a caller that sends a list, the list. +func pathsArg(a map[string]any) []string { + var raw []string + switch v := a["paths"].(type) { + case string: + raw = strings.Split(v, "\n") + case []any: + for _, p := range v { + if s, ok := p.(string); ok { + raw = append(raw, s) + } + } + } + var out []string + for _, p := range raw { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +// bounded is the last bound (novox/hq ADR 0310): whatever a verb made, no answer is larger than MaxAnswer. +func bounded(answer any, err error) (any, error) { + if err != nil { + return nil, err + } + body, err := json.Marshal(answer) + if err != nil { + return nil, err + } + if len(body) > MaxAnswer { + return nil, fmt.Errorf("the answer is %d bytes, and an answer of node-reads is at most %d: ask for less", len(body), MaxAnswer) + } + return answer, nil +} + +// verb is one of the seat's verbs: listed as `.`, so the runtime serves it on the seat's subject. +// The descriptions are the controller's (internal/catalogue/reads.go), said again here for a caller of the +// module's own address. +func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool { + return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, + Run: func(a map[string]any) (any, error) { return bounded(run(a)) }} +} + +func object(props map[string]any, required ...string) map[string]any { + if required == nil { + required = []string{} + } + return map[string]any{"type": "object", "properties": props, "required": required} +} + +func tools(r Reader) []stdio.Tool { + return []stdio.Tool{ + verb("stat", "Each component of one path or several, from / down: its type, owner and group (name and "+ + "number), mode, whether it is a link and to what, and whether it is a mount point and of which file "+ + "system. A link is not followed unless follow is \"true\". Read as the account mesh-reads, which holds no "+ + "secret: a component it may not reach says so. At most 200 paths in one call.", + object(map[string]any{"paths": str("one absolute path, or several separated by line breaks"), + "follow": str("\"true\": also say where the path leads once every link is followed")}, "paths"), + func(a map[string]any) (any, error) { return r.Stat(pathsArg(a), arg(a, "follow") == "true") }), + verb("readlink", "What one link points to, as written, and where the path leads once every link is "+ + "followed, and whether that exists.", + object(map[string]any{"path": str("an absolute path")}, "path"), + func(a map[string]any) (any, error) { return r.Readlink(arg(a, "path")) }), + verb("list", "Every entry of one directory with its type, owner, group, mode, size and, for a link, its "+ + "target. At most 2000 entries; a longer directory is cut and says so. A place a secret is kept is "+ + "refused by name.", + object(map[string]any{"path": str("an absolute path to a directory")}, "path"), + func(a map[string]any) (any, error) { return r.List(arg(a, "path")) }), + verb("read", "One text file, or a part of it by line: at most 256 KiB in one answer (64 KiB unless more "+ + "is asked), cut at a line and said so. A negative from counts from the end, as tail does. A binary file "+ + "is refused; a place a secret is kept, or a known credential file, is refused by name; a line shaped "+ + "like a secret is withheld and its number said. /proc and /sys are read as far as mesh-reads may.", + object(map[string]any{"path": str("an absolute path to a file"), + "from": str("the first line to answer, counted from 1; negative: that many lines from the end (optional)"), + "lines": str("how many lines to answer (optional)"), + "bytes": str("the most bytes to answer, up to 262144 (optional, 65536 unless given)")}, "path"), + func(a map[string]any) (any, error) { + var ask ReadAsk + var err error + if ask.From, err = number(a, "from"); err != nil { + return nil, err + } + if ask.Lines, err = number(a, "lines"); err != nil { + return nil, err + } + if ask.Bytes, err = number(a, "bytes"); err != nil { + return nil, err + } + return r.Read(arg(a, "path"), ask) + }), + verb("search", "Find a fixed string (never a pattern) in one file or under one directory: every matching "+ + "line with its file and number. Bounded: at most 500 matches (100 unless more is asked), 8 directories "+ + "deep (4 unless more is asked), 5000 files, 4 MiB per file and 20 seconds, on the file system of the path "+ + "named; links are not followed and binary files are skipped. A place a secret is kept is skipped and named; a matching line shaped like "+ + "a secret is withheld. The answer says which bound it met.", + object(map[string]any{"text": str("the fixed string to find"), + "path": str("an absolute path to a file or a directory"), + "depth": str("how many directories deep, up to 8 (optional)"), + "matches": str("the most matches to answer, up to 500 (optional)"), + "case": str("\"false\": match whatever the case (optional)")}, "text", "path"), + func(a map[string]any) (any, error) { + text, _ := a["text"].(string) // as given: a space at either end is part of what is found + ask := SearchAsk{Text: text, Path: arg(a, "path"), CaseInsensitive: arg(a, "case") == "false"} + var err error + if ask.Depth, err = number(a, "depth"); err != nil { + return nil, err + } + if ask.Matches, err = number(a, "matches"); err != nil { + return nil, err + } + return r.Search(ask) + }), + verb("account", "One account or group by name or number, as this node's account files say: an account's "+ + "number, group, home, shell and every group it is in; a group's number and members. With neither, every "+ + "account and group. Never a password field.", + object(map[string]any{"name": str("an account's name or number (optional)"), + "group": str("a group's name or number (optional)")}), + func(a map[string]any) (any, error) { return r.Account(arg(a, "name"), arg(a, "group")) }), + } +} diff --git a/modules/mesh-reads/cmd/mesh-reads/manifest_test.go b/modules/mesh-reads/cmd/mesh-reads/manifest_test.go new file mode 100644 index 00000000..2422aef5 --- /dev/null +++ b/modules/mesh-reads/cmd/mesh-reads/manifest_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +// ADR 0310 §3: the account the reads run as is the module's own, made by the mesh, with no login shell, no +// home of the operator's and no group beside its own, and the module runs as it. +func TestTheAccountHoldsNoSecretsGroupAndNoLogin(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + RunsAs string `json:"runs-as"` + SecretsOwner string `json:"secrets-owner"` + Claims []map[string]any `json:"claims"` + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.RunsAs != "mesh-reads" || m.SecretsOwner != "mesh-reads" { + t.Errorf("runs as %q, secrets owned by %q", m.RunsAs, m.SecretsOwner) + } + if len(m.Claims) != 1 || m.Claims[0]["name"] != Seat { + t.Errorf("claims %v", m.Claims) + } + found := false + for _, r := range m.Resources { + if r["type"] != "user" { + continue + } + found = true + if r["name"] != m.RunsAs { + t.Errorf("a user other than its own: %v", r) + } + if r["shell"] != "/usr/bin/nologin" { + t.Errorf("the account has a login shell: %v", r["shell"]) + } + if home, _ := r["home"].(string); !strings.HasPrefix(home, "/var/lib/") { + t.Errorf("the account's home is %q, not one of the mesh's", home) + } + if _, grouped := r["groups"]; grouped { + t.Errorf("the account is put in groups: %v", r["groups"]) + } + } + if !found { + t.Error("the module makes no account of its own") + } + for _, r := range m.Resources { + if r["type"] != "user" && r["type"] != "directory" { + t.Errorf("a read-only holder declares a %v: %v", r["type"], r) + } + } +} diff --git a/modules/mesh-reads/cmd/mesh-reads/reads.go b/modules/mesh-reads/cmd/mesh-reads/reads.go new file mode 100644 index 00000000..762a5de8 --- /dev/null +++ b/modules/mesh-reads/cmd/mesh-reads/reads.go @@ -0,0 +1,1043 @@ +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "time" +) + +// The bounds (novox/hq ADR 0310, Consequences: set by the build and stated in each verb's description). Every +// answer stays under MaxAnswer, which the tools check once more after an answer is made. +const ( + MaxAnswer = 1 << 20 // no answer of this module is larger, whatever the bounds below allow + MaxPaths = 200 // paths one stat answers + MaxComponents = 64 // components of one path + MaxEntries = 2000 // entries one list answers + DefaultReadSize = 64 << 10 // bytes a read answers unless more is asked + MaxReadSize = 256 << 10 // bytes a read answers at most + MaxScan = 64 << 20 // bytes a read passes over to reach the lines asked + DefaultMatches = 100 + MaxMatches = 500 + DefaultDepth = 4 + MaxDepth = 8 + MaxFiles = 5000 // files one search opens + MaxFileScan = 4 << 20 // bytes one search reads of one file + MaxLine = 300 // characters of one matching line answered + SearchTime = 20 * time.Second + MaxAccounts = 2000 +) + +// Reader reads one node's files and accounts as the account it runs as. +type Reader struct { + Refuser + // Passwd, Group and MountInfo are the files accounts, groups and mount points are read from. + Passwd, Group, MountInfo string + // Euid is the account this runs as; never root (ADR 0310 §3). + Euid func() int + // Now is the clock a search is bounded by. + Now func() time.Time +} + +func newReader(places []string) Reader { + return Reader{Refuser: Refuser{Places: places}, Passwd: "/etc/passwd", Group: "/etc/group", + MountInfo: "/proc/self/mountinfo", Euid: os.Geteuid, Now: time.Now} +} + +// asItsOwn refuses every read when this runs as root: what it may see is bounded by its account's file +// modes, and root's are none (novox/hq ADR 0310 §3). Loud, never a silent widening. +func (r Reader) asItsOwn() error { + if r.Euid() == 0 { + return errors.New("mesh-reads runs as root here, and its reads are bounded by an account of its own that " + + "holds no secret (novox/hq ADR 0310 §3): it answers nothing until it runs as mesh-reads") + } + return nil +} + +func absolute(path string) (string, error) { + path = strings.TrimSpace(path) + if !strings.HasPrefix(path, "/") { + return "", fmt.Errorf("%q is not an absolute path: name it from /", path) + } + return filepath.Clean(path), nil +} + +// why turns a file error into the words an answer says: what stopped the read, never a Go error's text. +func why(err error) string { + switch { + case errors.Is(err, fs.ErrNotExist): + return "it does not exist" + case errors.Is(err, fs.ErrPermission): + return "the account mesh-reads may not reach it (its file modes refuse it)" + case errors.Is(err, syscall.ELOOP): + return "its links loop" + case errors.Is(err, syscall.ENOTDIR): + return "a component of it is not a directory" + } + var pe *fs.PathError + if errors.As(err, &pe) { + return pe.Err.Error() + } + return err.Error() +} + +// errRefusedByName marks a refusal by name, which a search names rather than counts as unreadable. +var errRefusedByName = errors.New("refused by name") + +// openJudged opens a path and judges what was opened, not its name (the review of #244: a link flipped between +// the name check and the open read a secret place 3744 times in 20000): the type is checked before the open, so +// a device or a fifo is never opened; the descriptor is opened non-blocking, without a controlling terminal, and +// in a search without following a link; then it is checked again by fstat, and the path the kernel says it is +// (/proc/self/fd) is judged by name like any path asked. A deleted file, or one the kernel names no path for, is +// refused. +func (r Reader) openJudged(path string, nofollow, dir bool) (*os.File, fs.FileInfo, error) { + stat := os.Stat + if nofollow { + stat = os.Lstat + } + before, err := stat(path) + if err != nil { + return nil, nil, fmt.Errorf("%s cannot be read: %s", path, why(err)) + } + if err := wantType(path, before, dir); err != nil { + return nil, nil, err + } + flags := os.O_RDONLY | syscall.O_NONBLOCK | syscall.O_NOCTTY | syscall.O_CLOEXEC + if nofollow { + flags |= syscall.O_NOFOLLOW + } + if dir { + flags |= syscall.O_DIRECTORY + } + f, err := os.OpenFile(path, flags, 0) + if err != nil { + return nil, nil, fmt.Errorf("%s cannot be read: %s", path, why(err)) + } + info, err := f.Stat() + if err == nil { + err = wantType(path, info, dir) + } + if err != nil { + f.Close() + return nil, nil, err + } + real, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", f.Fd())) + if err != nil || !strings.HasPrefix(real, "/") || strings.HasSuffix(real, " (deleted)") { + f.Close() + return nil, nil, fmt.Errorf("%s is refused: the kernel names no path for what was opened, so it cannot be "+ + "judged by name (novox/hq ADR 0310 §5)", path) + } + if why := r.Refused(real); why != "" { + f.Close() + return nil, nil, fmt.Errorf("%s is %s, which is refused by name: %s (novox/hq ADR 0310 §5): %w", path, real, why, errRefusedByName) + } + return f, info, nil +} + +func wantType(path string, info fs.FileInfo, dir bool) error { + switch { + case dir && !info.IsDir(): + return fmt.Errorf("%s is a %s, not a directory", path, typeOf(info.Mode())) + case !dir && !info.Mode().IsRegular(): + return fmt.Errorf("%s is a %s, not a file: list a directory, stat anything else", path, typeOf(info.Mode())) + } + return nil +} + +// deviceOf is the file system a path is on. +func deviceOf(path string) (uint64, bool) { + info, err := os.Lstat(path) + if err != nil { + return 0, false + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return uint64(st.Dev), true +} + +// ---- stat ---- + +// Component is one component of a path as lstat sees it. +type Component struct { + Path string `json:"path"` + Type string `json:"type,omitempty"` + Owner string `json:"owner,omitempty"` + UID *int `json:"uid,omitempty"` + Group string `json:"group,omitempty"` + GID *int `json:"gid,omitempty"` + Mode string `json:"mode,omitempty"` // octal, with the setuid, setgid and sticky bits + Perms string `json:"perms,omitempty"` // as ls shows them + Size *int64 `json:"size,omitempty"` + Link string `json:"link,omitempty"` // what a link points to, as written + Mount *Mount `json:"mount,omitempty"` + Secret string `json:"secret-place,omitempty"` // why a read under it is refused + Unreachd string `json:"unreachable,omitempty"` +} + +// Mount is a mount point: its file system and source. +type Mount struct { + FS string `json:"fs"` + Source string `json:"source"` +} + +// PathStat is one path's components, from / down, and where it leads once every link is followed. +type PathStat struct { + Path string `json:"path"` + Components []Component `json:"components"` + LeadsTo string `json:"leads-to,omitempty"` + Leads *Component `json:"there,omitempty"` + Stopped string `json:"stopped,omitempty"` +} + +// StatAnswer is what stat answers. +type StatAnswer struct { + Paths []PathStat `json:"paths"` + Withheld []Withheld `json:"withheld,omitempty"` +} + +func (r Reader) Stat(paths []string, follow bool) (StatAnswer, error) { + if err := r.asItsOwn(); err != nil { + return StatAnswer{}, err + } + if len(paths) == 0 { + return StatAnswer{}, errors.New("name a path to stat") + } + if len(paths) > MaxPaths { + return StatAnswer{}, fmt.Errorf("%d paths asked, and one stat answers at most %d: ask in parts", len(paths), MaxPaths) + } + names := r.names() + mounts := r.mounts() + var out StatAnswer + for _, raw := range paths { + path, err := absolute(raw) + if err != nil { + return StatAnswer{}, err + } + ps := PathStat{Path: path} + walk := []string{"/"} + acc := "" + for _, part := range strings.Split(strings.TrimPrefix(path, "/"), "/") { + if part == "" { + continue + } + acc += "/" + part + walk = append(walk, acc) + } + if len(walk) > MaxComponents { + return StatAnswer{}, fmt.Errorf("%s has %d components, and a stat answers at most %d", path, len(walk), MaxComponents) + } + for _, p := range walk { + c := r.component(p, names, mounts) + ps.Components = append(ps.Components, c) + if c.Unreachd != "" { + ps.Stopped = c.Path + ": " + c.Unreachd + break + } + } + if follow && ps.Stopped == "" { + if leads, err := filepath.EvalSymlinks(path); err != nil { + ps.Stopped = "following its links: " + why(err) + } else { + ps.LeadsTo = leads + if leads != path { + c := r.component(leads, names, mounts) + ps.Leads = &c + } + } + } + out.Paths = append(out.Paths, ps) + } + out.Withheld = cleanStat(&out) + return out, nil +} + +func (r Reader) component(p string, names accounts, mounts map[string]Mount) Component { + c := Component{Path: p} + info, err := os.Lstat(p) + if err != nil { + c.Unreachd = why(err) + return c + } + c.Type = typeOf(info.Mode()) + c.Perms = info.Mode().String() + if st, ok := info.Sys().(*syscall.Stat_t); ok { + uid, gid := int(st.Uid), int(st.Gid) + c.UID, c.GID = &uid, &gid + c.Owner, c.Group = names.user(uid), names.group(gid) + c.Mode = fmt.Sprintf("%04o", st.Mode&07777) + } + if info.Mode().IsRegular() { + size := info.Size() + c.Size = &size + } + if info.Mode()&fs.ModeSymlink != 0 { + if target, err := os.Readlink(p); err == nil { + c.Link = target + } + } + if m, ok := mounts[p]; ok { + c.Mount = &m + } + if why := r.Refused(p); why != "" { + c.Secret = why + } + return c +} + +func typeOf(m fs.FileMode) string { + switch { + case m.IsDir(): + return "directory" + case m&fs.ModeSymlink != 0: + return "link" + case m.IsRegular(): + return "file" + case m&fs.ModeNamedPipe != 0: + return "fifo" + case m&fs.ModeSocket != 0: + return "socket" + case m&fs.ModeCharDevice != 0: + return "character device" + case m&fs.ModeDevice != 0: + return "block device" + } + return "other" +} + +func cleanStat(a *StatAnswer) []Withheld { + var w []Withheld + cleanC := func(c *Component) { + for _, f := range []*string{&c.Link} { + if s, what := clean(*f); what != "" { + w = append(w, Withheld{At: "a link's target", Class: what}) + *f = s + } + } + if c.Mount != nil { + if s, what := clean(c.Mount.Source); what != "" { + w = append(w, Withheld{At: "a mount's source", Class: what}) + c.Mount.Source = s + } + } + } + for i := range a.Paths { + p := &a.Paths[i] + for j := range p.Components { + cleanC(&p.Components[j]) + } + if p.Leads != nil { + cleanC(p.Leads) + } + } + return w +} + +// ---- readlink ---- + +// LinkAnswer is what readlink answers. +type LinkAnswer struct { + Path string `json:"path"` + Link string `json:"link,omitempty"` // as written; empty when the path is not a link + IsLink bool `json:"is-link"` + LeadsTo string `json:"leads-to,omitempty"` + Exists bool `json:"exists"` + Stopped string `json:"stopped,omitempty"` + Withheld []Withheld `json:"withheld,omitempty"` +} + +func (r Reader) Readlink(raw string) (LinkAnswer, error) { + if err := r.asItsOwn(); err != nil { + return LinkAnswer{}, err + } + path, err := absolute(raw) + if err != nil { + return LinkAnswer{}, err + } + a := LinkAnswer{Path: path} + info, err := os.Lstat(path) + if err != nil { + a.Stopped = why(err) + return a, nil + } + if info.Mode()&fs.ModeSymlink != 0 { + a.IsLink = true + if a.Link, err = os.Readlink(path); err != nil { + a.Stopped = why(err) + } + } + if leads, err := filepath.EvalSymlinks(path); err == nil { + a.LeadsTo, a.Exists = leads, true + } else if a.Stopped == "" { + a.Stopped = "following its links: " + why(err) + } + for _, f := range []*string{&a.Link} { + if s, what := clean(*f); what != "" { + a.Withheld = append(a.Withheld, Withheld{At: "a link's target", Class: what}) + *f = s + } + } + return a, nil +} + +// ---- list ---- + +// Entry is one entry of a directory. +type Entry struct { + Name string `json:"name"` + Type string `json:"type"` + Owner string `json:"owner,omitempty"` + Group string `json:"group,omitempty"` + Mode string `json:"mode,omitempty"` + Size *int64 `json:"size,omitempty"` + Link string `json:"link,omitempty"` +} + +// ListAnswer is what list answers. +type ListAnswer struct { + Path string `json:"path"` + Entries []Entry `json:"entries"` + Count int `json:"count"` + Cut string `json:"cut,omitempty"` + Withheld []Withheld `json:"withheld,omitempty"` +} + +// refusedRead is a refusal by name, before any read: the path asked and where it leads are both judged, so a +// link to a secret place is refused as the place is. +func (r Reader) refusedRead(path string) error { + if why := r.Refused(path); why != "" { + return fmt.Errorf("%s is refused by name: %s (novox/hq ADR 0310 §5)", path, why) + } + if leads, err := filepath.EvalSymlinks(path); err == nil && leads != path { + if why := r.Refused(leads); why != "" { + return fmt.Errorf("%s leads to %s, which is refused by name: %s (novox/hq ADR 0310 §5)", path, leads, why) + } + } + return nil +} + +func (r Reader) List(raw string) (ListAnswer, error) { + if err := r.asItsOwn(); err != nil { + return ListAnswer{}, err + } + path, err := absolute(raw) + if err != nil { + return ListAnswer{}, err + } + if err := r.refusedRead(path); err != nil { + return ListAnswer{}, err + } + dir, _, err := r.openJudged(path, false, true) + if err != nil { + return ListAnswer{}, err + } + entries, err := dir.ReadDir(-1) + dir.Close() + if err != nil { + return ListAnswer{}, fmt.Errorf("%s cannot be listed: %s", path, why(err)) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + names := r.names() + a := ListAnswer{Path: path, Count: len(entries), Entries: []Entry{}} + if len(entries) > MaxEntries { + a.Cut = fmt.Sprintf("cut at %d of %d entries (the bound of one list), in name order", MaxEntries, len(entries)) + entries = entries[:MaxEntries] + } + for _, e := range entries { + en := Entry{Name: e.Name(), Type: typeOf(e.Type())} + if info, err := e.Info(); err == nil { + en.Type = typeOf(info.Mode()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + en.Owner, en.Group = names.user(int(st.Uid)), names.group(int(st.Gid)) + en.Mode = fmt.Sprintf("%04o", st.Mode&07777) + } + if info.Mode().IsRegular() { + size := info.Size() + en.Size = &size + } + if info.Mode()&fs.ModeSymlink != 0 { + en.Link, _ = os.Readlink(filepath.Join(path, e.Name())) + } + } + for _, f := range []*string{&en.Link} { + if s, what := clean(*f); what != "" { + a.Withheld = append(a.Withheld, Withheld{At: "a link's target", Class: what}) + *f = s + } + } + a.Entries = append(a.Entries, en) + } + return a, nil +} + +// ---- read ---- + +// ReadAnswer is what read answers. +type ReadAnswer struct { + Path string `json:"path"` + Size int64 `json:"size"` // as the file says; 0 for most of /proc and /sys + From int `json:"from"` // the first line answered, counted from 1 + Lines int `json:"lines"` + Text string `json:"text"` + Cut string `json:"cut,omitempty"` + Withheld []int `json:"withheld-lines,omitempty"` + Why string `json:"withheld-because,omitempty"` +} + +// ReadAsk is what a read asks: the lines from From (1 is the first; a negative From counts from the end, as +// tail does), at most Lines of them (0: as many as fit), and at most Bytes bytes. +type ReadAsk struct { + From, Lines, Bytes int +} + +func (r Reader) Read(raw string, ask ReadAsk) (ReadAnswer, error) { + if err := r.asItsOwn(); err != nil { + return ReadAnswer{}, err + } + path, err := absolute(raw) + if err != nil { + return ReadAnswer{}, err + } + if err := r.refusedRead(path); err != nil { + return ReadAnswer{}, err + } + bound := ask.Bytes + switch { + case bound <= 0: + bound = DefaultReadSize + case bound > MaxReadSize: + return ReadAnswer{}, fmt.Errorf("%d bytes asked, and one read answers at most %d", bound, MaxReadSize) + } + if ask.Lines < 0 { + return ReadAnswer{}, errors.New("lines is a count: name how many, or none for as many as fit") + } + f, info, err := r.openJudged(path, false, false) + if err != nil { + return ReadAnswer{}, err + } + defer f.Close() + a := ReadAnswer{Path: path, Size: info.Size()} + lines, cut, err := pick(io.LimitReader(f, MaxScan), ask, bound) + if err != nil { + return ReadAnswer{}, fmt.Errorf("%s: %v", path, err) + } + a.Cut = cut + if len(lines) > 0 { + a.From = lines[0].n + } + var text strings.Builder + for _, l := range lines { + s, what := clean(l.text) + if l.block { + s, what = "‹withheld: a key or certificate block›", "a key or certificate block" + } + if what != "" { + a.Withheld = append(a.Withheld, l.n) + a.Why = "a line shaped like a secret is withheld by the content rule (novox/hq ADR 0310 §4)" + } + text.WriteString(s) + if l.newline { + text.WriteByte('\n') + } + } + a.Text, a.Lines = text.String(), len(lines) + return a, nil +} + +type line struct { + n int + text string + newline bool + size int + block bool // inside a BEGIN … END block, whose every line is withheld +} + +// pick reads lines and keeps the ones asked, within bound bytes. A NUL byte is a binary file, refused. +func pick(in io.Reader, ask ReadAsk, bound int) ([]line, string, error) { + br := bufio.NewReaderSize(in, 64<<10) + var kept []line + size, n := 0, 0 + from := ask.From + if from == 0 { + from = 1 + } + var blocks blockState + for { + raw, err := br.ReadString('\n') + if raw == "" && err != nil { + if err == io.EOF { + break + } + return nil, "", err + } + n++ + if strings.IndexByte(raw, 0) >= 0 { + return nil, "", errors.New("it is a binary file, and read answers text") + } + l := line{n: n, text: strings.TrimSuffix(raw, "\n"), newline: strings.HasSuffix(raw, "\n"), size: len(raw), + block: blocks.in(raw)} + if from > 0 { + if n < from { + continue + } + if ask.Lines > 0 && len(kept) == ask.Lines { + break + } + if size+len(raw) > bound { + return kept, fmt.Sprintf("cut at line %d: the answer holds at most %d bytes", n, bound), nil + } + kept = append(kept, l) + size += len(raw) + } else { + // From the end, as tail does: the last -from lines, kept as they pass. + kept = append(kept, l) + size += len(raw) + for len(kept) > -from || size > bound { + size -= kept[0].size + kept = kept[1:] + } + } + if err == io.EOF { + break + } + } + if from > 0 && n < from && n > 0 { + return nil, "", fmt.Errorf("it has %d lines, and line %d was asked", n, from) + } + if from < 0 && ask.Lines > 0 && len(kept) > ask.Lines { + kept = kept[:ask.Lines] + } + return kept, "", nil +} + +// ---- search ---- + +// Match is one matching line. +type Match struct { + File string `json:"file"` + Line int `json:"line"` + Text string `json:"text"` +} + +// SearchAnswer is what search answers. +type SearchAnswer struct { + Text string `json:"text"` + Path string `json:"path"` + Matches []Match `json:"matches"` + Files int `json:"files-read"` + Bound string `json:"bound-met,omitempty"` + Skipped []string `json:"refused-by-name,omitempty"` + Unreadable int `json:"unreadable,omitempty"` + OtherFS []string `json:"other-file-systems-skipped,omitempty"` + Binary int `json:"binary-skipped,omitempty"` + Withheld []Withheld `json:"withheld,omitempty"` +} + +// SearchAsk is what a search asks. +type SearchAsk struct { + Text, Path string + Depth, Matches int + CaseInsensitive bool +} + +var errBound = errors.New("bound met") + +func (r Reader) Search(ask SearchAsk) (SearchAnswer, error) { + if err := r.asItsOwn(); err != nil { + return SearchAnswer{}, err + } + if ask.Text == "" { + return SearchAnswer{}, errors.New("name the text to find") + } + if strings.ContainsAny(ask.Text, "\n\x00") { + return SearchAnswer{}, errors.New("the text is found within one line: it may not hold a line break") + } + root, err := absolute(ask.Path) + if err != nil { + return SearchAnswer{}, err + } + if err := r.refusedRead(root); err != nil { + return SearchAnswer{}, err + } + depth, matches := ask.Depth, ask.Matches + switch { + case depth == 0: + depth = DefaultDepth + case depth < 0 || depth > MaxDepth: + return SearchAnswer{}, fmt.Errorf("depth %d asked; a search goes at most %d directories deep", depth, MaxDepth) + } + switch { + case matches == 0: + matches = DefaultMatches + case matches < 0 || matches > MaxMatches: + return SearchAnswer{}, fmt.Errorf("%d matches asked; a search answers at most %d", matches, MaxMatches) + } + needle := ask.Text + if ask.CaseInsensitive { + needle = strings.ToLower(needle) + } + a := SearchAnswer{Text: ask.Text, Path: root, Matches: []Match{}} + deadline := r.Now().Add(SearchTime) + rootDev, hasDev := deviceOf(root) + base := strings.Count(root, "/") + if root == "/" { + base = 0 + } + err = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if r.Now().After(deadline) { + a.Bound = fmt.Sprintf("time: a search runs at most %s", SearchTime) + return errBound + } + if err != nil { + a.Unreadable++ + if d != nil && d.IsDir() { + return fs.SkipDir + } + return nil + } + if p != root { + if why := r.Refused(p); why != "" { + a.Skipped = append(a.Skipped, p) + if d.IsDir() { + return fs.SkipDir + } + return nil + } + } + if d.IsDir() { + if p != root && strings.Count(p, "/")-base > depth { + return fs.SkipDir + } + // The root's file system only, as grep -r with find's -xdev: a hard network mount would hang a + // read past every bound, and /proc and /sys are searched only from within them. + if dev, ok := deviceOf(p); p != root && hasDev && ok && dev != rootDev { + a.OtherFS = append(a.OtherFS, p) + return fs.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil // links are not followed; a device, a fifo or a socket is not text + } + if a.Files >= MaxFiles { + a.Bound = fmt.Sprintf("files: a search reads at most %d files", MaxFiles) + return errBound + } + a.Files++ + return r.searchFile(p, needle, ask.CaseInsensitive, matches, deadline, &a) + }) + if err != nil && !errors.Is(err, errBound) { + return SearchAnswer{}, fmt.Errorf("%s cannot be searched: %s", root, why(err)) + } + return a, nil +} + +func (r Reader) searchFile(p, needle string, fold bool, bound int, deadline time.Time, a *SearchAnswer) error { + f, _, err := r.openJudged(p, true, false) + if err != nil { + if errors.Is(err, errRefusedByName) { + a.Skipped = append(a.Skipped, p) + } else { + a.Unreadable++ + } + return nil + } + defer f.Close() + br := bufio.NewReaderSize(io.LimitReader(f, MaxFileScan), 64<<10) + if head, _ := br.Peek(8 << 10); bytes.IndexByte(head, 0) >= 0 { + a.Binary++ + return nil + } + var blocks blockState + n := 0 + for { + raw, err := br.ReadString('\n') + if raw == "" && err != nil { + return nil + } + n++ + if n%1000 == 0 && r.Now().After(deadline) { + a.Bound = fmt.Sprintf("time: a search runs at most %s", SearchTime) + return errBound + } + block := blocks.in(raw) + hay := raw + if fold { + hay = strings.ToLower(hay) + } + if strings.Contains(hay, needle) { + if len(a.Matches) >= bound { + a.Bound = fmt.Sprintf("matches: %d answered, as asked or the default", bound) + return errBound + } + text := strings.TrimRight(raw, "\r\n") + if r := []rune(text); len(r) > MaxLine { + text = string(r[:MaxLine]) + "…" + } + if s, what := clean(text); what != "" || block { + if block { + s, what = "‹withheld: a key or certificate block›", "a key or certificate block" + } + a.Withheld = append(a.Withheld, Withheld{At: "a matching line", Class: what}) + text = s + } + a.Matches = append(a.Matches, Match{File: p, Line: n, Text: text}) + } + if err != nil { + return nil + } + } +} + +// blockState follows BEGIN … END blocks through a file (the review of #244): a private key's body is base64 a +// line at a time, and one line of it can pass the content rule alone, so every line from a BEGIN through its +// END is withheld — a certificate's too, which costs nothing a read needs. +type blockState struct{ open bool } + +func (b *blockState) in(raw string) bool { + switch { + case strings.Contains(raw, "-----BEGIN "): + b.open = !strings.Contains(raw, "-----END ") + return true + case b.open: + if strings.Contains(raw, "-----END ") { + b.open = false + } + return true + } + return false +} + +// ---- accounts ---- + +// Account is one account as the account file says it, never its password field. +type Account struct { + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` + Group string `json:"group,omitempty"` + Gecos string `json:"gecos,omitempty"` + Home string `json:"home"` + Shell string `json:"shell"` + Groups []string `json:"groups,omitempty"` +} + +// GroupEntry is one group. +type GroupEntry struct { + Name string `json:"name"` + GID int `json:"gid"` + Members []string `json:"members"` +} + +// AccountAnswer is what account answers. +type AccountAnswer struct { + Accounts []Account `json:"accounts,omitempty"` + Groups []GroupEntry `json:"groups,omitempty"` + Source string `json:"source"` + Cut string `json:"cut,omitempty"` + Withheld []Withheld `json:"withheld,omitempty"` +} + +type accounts struct { + users []Account + groups []GroupEntry +} + +func (a accounts) user(uid int) string { + for _, u := range a.users { + if u.UID == uid { + return u.Name + } + } + return "" +} + +func (a accounts) group(gid int) string { + for _, g := range a.groups { + if g.GID == gid { + return g.Name + } + } + return "" +} + +// names reads the account files; a file it cannot read gives no names, and a component says only numbers. +func (r Reader) names() accounts { + var a accounts + if raw, err := os.ReadFile(r.Passwd); err == nil { + for _, l := range strings.Split(string(raw), "\n") { + f := strings.Split(l, ":") + if len(f) < 7 || strings.HasPrefix(l, "#") { + continue + } + uid, e1 := strconv.Atoi(f[2]) + gid, e2 := strconv.Atoi(f[3]) + if e1 != nil || e2 != nil { + continue + } + a.users = append(a.users, Account{Name: f[0], UID: uid, GID: gid, Gecos: f[4], Home: f[5], Shell: f[6]}) + } + } + if raw, err := os.ReadFile(r.Group); err == nil { + for _, l := range strings.Split(string(raw), "\n") { + f := strings.Split(l, ":") + if len(f) < 4 || strings.HasPrefix(l, "#") { + continue + } + gid, err := strconv.Atoi(f[2]) + if err != nil { + continue + } + members := []string{} + for _, m := range strings.Split(f[3], ",") { + if m = strings.TrimSpace(m); m != "" { + members = append(members, m) + } + } + a.groups = append(a.groups, GroupEntry{Name: f[0], GID: gid, Members: members}) + } + } + return a +} + +func (r Reader) Account(name, group string) (AccountAnswer, error) { + if err := r.asItsOwn(); err != nil { + return AccountAnswer{}, err + } + all := r.names() + if len(all.users) == 0 { + return AccountAnswer{}, fmt.Errorf("%s gave no account: it is unreadable or empty", r.Passwd) + } + out := AccountAnswer{Source: r.Passwd + ", " + r.Group + " (an account kept elsewhere, such as a directory " + + "service or systemd's dynamic accounts, is not here)"} + complete := func(u Account) Account { + u.Group = all.group(u.GID) + for _, g := range all.groups { + if g.GID == u.GID { + u.Groups = append(u.Groups, g.Name) + continue + } + for _, m := range g.Members { + if m == u.Name { + u.Groups = append(u.Groups, g.Name) + } + } + } + return u + } + name, group = strings.TrimSpace(name), strings.TrimSpace(group) + switch { + case name != "": + for _, u := range all.users { + if u.Name == name || strconv.Itoa(u.UID) == name { + out.Accounts = append(out.Accounts, complete(u)) + } + } + if len(out.Accounts) == 0 { + return AccountAnswer{}, fmt.Errorf("no account %q on this node", name) + } + case group != "": + for _, g := range all.groups { + if g.Name == group || strconv.Itoa(g.GID) == group { + // Its members as getent says them, and every account whose own group it is. + for _, u := range all.users { + if u.GID == g.GID && !contains(g.Members, u.Name) { + g.Members = append(g.Members, u.Name) + } + } + out.Groups = append(out.Groups, g) + } + } + if len(out.Groups) == 0 { + return AccountAnswer{}, fmt.Errorf("no group %q on this node", group) + } + default: + for _, u := range all.users { + out.Accounts = append(out.Accounts, complete(u)) + } + out.Groups = all.groups + if len(out.Accounts) > MaxAccounts { + out.Cut = fmt.Sprintf("cut at %d of %d accounts", MaxAccounts, len(out.Accounts)) + out.Accounts = out.Accounts[:MaxAccounts] + } + if len(out.Groups) > MaxAccounts { + out.Cut = strings.TrimPrefix(out.Cut+"; ", "; ") + fmt.Sprintf("cut at %d of %d groups", MaxAccounts, len(out.Groups)) + out.Groups = out.Groups[:MaxAccounts] + } + } + for i := range out.Accounts { + u := &out.Accounts[i] + for _, f := range []*string{&u.Gecos, &u.Home, &u.Shell} { + if s, what := clean(*f); what != "" { + out.Withheld = append(out.Withheld, Withheld{At: "account " + u.Name, Class: what}) + *f = s + } + } + } + sort.SliceStable(out.Accounts, func(i, j int) bool { return out.Accounts[i].UID < out.Accounts[j].UID }) + return out, nil +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// ---- mounts ---- + +// mounts reads the mount points this process sees, by path. /proc/self/mountinfo is readable by every account. +func (r Reader) mounts() map[string]Mount { + out := map[string]Mount{} + raw, err := os.ReadFile(r.MountInfo) + if err != nil { + return out + } + for _, l := range strings.Split(string(raw), "\n") { + f := strings.Fields(l) + dash := -1 + for i, x := range f { + if x == "-" { + dash = i + break + } + } + if len(f) < 5 || dash < 0 || dash+2 >= len(f) { + continue + } + out[unescape(f[4])] = Mount{FS: f[dash+1], Source: unescape(f[dash+2])} + } + return out +} + +// unescape reads mountinfo's octal escapes (\040 for a space). +func unescape(s string) string { + if !strings.Contains(s, `\`) { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+3 < len(s) { + if v, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil { + b.WriteByte(byte(v)) + i += 3 + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} diff --git a/modules/mesh-reads/cmd/mesh-reads/reads_test.go b/modules/mesh-reads/cmd/mesh-reads/reads_test.go new file mode 100644 index 00000000..461892e8 --- /dev/null +++ b/modules/mesh-reads/cmd/mesh-reads/reads_test.go @@ -0,0 +1,562 @@ +package main + +import ( + crand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + "testing" + "time" +) + +// A token shaped as a secret, written into files the tests read. +const aToken = "ghp_abcdefghijklmnopqrstuvwxyz0123456789" + +func testReader(t *testing.T, places ...string) (Reader, string) { + t.Helper() + dir := t.TempDir() + passwd := filepath.Join(dir, "passwd") + group := filepath.Join(dir, "group") + mountinfo := filepath.Join(dir, "mountinfo") + write(t, passwd, "root:x:0:0::/root:/usr/bin/bash\nops:x:1000:1000:The Operator:/home/ops:/usr/bin/zsh\n"+ + "mesh-reads:x:961:961::/var/lib/mesh-reads:/usr/bin/nologin\n") + write(t, group, "root:x:0:\nwheel:x:998:ops\nops:x:1000:\nmesh-reads:x:961:\ndocker:x:970:ops,builder\n") + data := filepath.Join(dir, "data") + write(t, mountinfo, "22 1 0:21 / / rw,relatime - btrfs /dev/nvme0n1p2 rw\n"+ + "40 22 0:40 / "+strings.ReplaceAll(data, " ", `\040`)+" rw - nfs4 home-server:/tank/data rw\n") + r := Reader{Refuser: Refuser{Places: places}, Passwd: passwd, Group: group, MountInfo: mountinfo, + Euid: func() int { return 961 }, Now: time.Now} + return r, dir +} + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// ADR 0310 §3: run as root, every read is refused, loudly — its bound is an account of its own. +func TestEveryReadIsRefusedAsRoot(t *testing.T) { + r, dir := testReader(t) + r.Euid = func() int { return 0 } + write(t, filepath.Join(dir, "f"), "x\n") + if _, err := r.Read(filepath.Join(dir, "f"), ReadAsk{}); err == nil || !strings.Contains(err.Error(), "root") { + t.Errorf("a read as root: %v", err) + } + if _, err := r.Stat([]string{dir}, false); err == nil { + t.Error("a stat as root was answered") + } + if _, err := r.Account("ops", ""); err == nil { + t.Error("an account lookup as root was answered") + } +} + +// ADR 0310 §5: without the controller's list of secret places it does not start: a refusal it promises would be +// silent. +func TestItDoesNotStartWithoutTheNodesSecretPlaces(t *testing.T) { + if _, err := placesFrom(""); err == nil || !strings.Contains(err.Error(), PlacesWord) { + t.Errorf("no places: %v", err) + } + if p, err := placesFrom(`["/var/lib/mesh-vault"]`); err != nil || len(p) != 1 { + t.Errorf("a list: %v %v", p, err) + } + if p, err := placesFrom(`[]`); err != nil || len(p) != 0 { + t.Errorf("an empty list is a node with no secret place: %v %v", p, err) + } +} + +// ADR 0310 §5: a path under the vault's directory, a module's own-secret directory and a known credential file +// are each refused by name before any read — though the account's modes would allow it — and a link to one is +// refused as the place is. +func TestSecretPlacesAreRefusedByNameBeforeAnyRead(t *testing.T) { + r0, dir := testReader(t) + vault := filepath.Join(dir, "var/lib/mesh-vault") + own := filepath.Join(dir, "var/lib/desk-channel") + r, _ := r0, dir + r.Places = []string{vault, own} + for _, p := range []string{ + filepath.Join(vault, "ledger/entries.json"), + filepath.Join(own, "broker"), + filepath.Join(own, "settings.json"), + filepath.Join(dir, "home/ops/.claude/.credentials.json"), + filepath.Join(dir, "home/ops/.ssh/config"), + filepath.Join(dir, "home/ops/.pgpass"), + filepath.Join(dir, "etc/ssh/ssh_host_ed25519_key"), + filepath.Join(dir, "etc/shadow"), + filepath.Join(dir, "srv/tls/server.key"), + } { + write(t, p, "harmless words\n") + if _, err := r.Read(p, ReadAsk{}); err == nil || !strings.Contains(err.Error(), "refused by name") { + t.Errorf("%s was read: %v", p, err) + } + } + if _, err := r.List(vault); err == nil || !strings.Contains(err.Error(), "refused by name") { + t.Errorf("the vault's directory was listed: %v", err) + } + link := filepath.Join(dir, "tmp/innocent") + write(t, filepath.Join(dir, "tmp/x"), "") + if err := os.Symlink(filepath.Join(own, "broker"), link); err != nil { + t.Fatal(err) + } + if _, err := r.Read(link, ReadAsk{}); err == nil || !strings.Contains(err.Error(), "leads to") { + t.Errorf("a link to a secret place was read: %v", err) + } + // A search under a directory holding a secret place skips it and names it. + write(t, filepath.Join(dir, "home/ops/notes.txt"), "harmless words\n") + a, err := r.Search(SearchAsk{Text: "harmless", Path: filepath.Join(dir, "home"), Depth: 8}) + if err != nil { + t.Fatal(err) + } + if len(a.Matches) != 1 || !strings.HasSuffix(a.Matches[0].File, "notes.txt") || len(a.Skipped) != 3 { + t.Errorf("search: matches %v, skipped %v", a.Matches, a.Skipped) + } + // What a public key or a certificate is, is not refused. + for _, p := range []string{filepath.Join(dir, "etc/ssh/ssh_host_ed25519_key.pub"), filepath.Join(dir, "etc/ssl/certs/ca.pem")} { + write(t, p, "harmless words\n") + if _, err := r.Read(p, ReadAsk{}); err != nil { + t.Errorf("%s was refused: %v", p, err) + } + } +} + +// ADR 0310 §3: what the account's file modes refuse is refused — the first bound, below the names. +func TestAFileTheAccountMayNotReadIsRefusedByItsMode(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("run as root, a file's mode refuses nothing") + } + r, dir := testReader(t) + p := filepath.Join(dir, "operators-only") + write(t, p, "the operator's words\n") + if err := os.Chmod(p, 0o000); err != nil { + t.Fatal(err) + } + if _, err := r.Read(p, ReadAsk{}); err == nil || !strings.Contains(err.Error(), "may not reach it") { + t.Errorf("a file of mode 0000: %v", err) + } + closed := filepath.Join(dir, "closed") + write(t, filepath.Join(closed, "inside"), "x") + if err := os.Chmod(closed, 0o700&^0o100); err != nil { + t.Fatal(err) + } + defer os.Chmod(closed, 0o755) + a, err := r.Stat([]string{filepath.Join(closed, "inside")}, false) + if err != nil { + t.Fatal(err) + } + if got := a.Paths[0]; !strings.Contains(got.Stopped, "may not reach") { + t.Errorf("a path through a directory it may not search: %+v", got) + } +} + +// ADR 0310 §2 and the bounds: a read over the size bound is cut at a line and says so; more than the most is +// refused; lines and a tail are answered as asked. +func TestAReadIsBounded(t *testing.T) { + r, dir := testReader(t) + p := filepath.Join(dir, "big") + var b strings.Builder + for i := 1; i <= 20000; i++ { + fmt.Fprintf(&b, "line %d of the file\n", i) + } + write(t, p, b.String()) + a, err := r.Read(p, ReadAsk{}) + if err != nil { + t.Fatal(err) + } + if len(a.Text) > DefaultReadSize || !strings.Contains(a.Cut, fmt.Sprint(DefaultReadSize)) || a.From != 1 { + t.Errorf("a default read: %d bytes, cut %q", len(a.Text), a.Cut) + } + if _, err := r.Read(p, ReadAsk{Bytes: MaxReadSize + 1}); err == nil { + t.Error("more than the most a read answers was answered") + } + a, err = r.Read(p, ReadAsk{From: 10, Lines: 2}) + if err != nil || a.Text != "line 10 of the file\nline 11 of the file\n" || a.From != 10 || a.Cut != "" { + t.Errorf("lines 10 and 11: %+v %v", a, err) + } + a, err = r.Read(p, ReadAsk{From: -2}) + if err != nil || a.Text != "line 19999 of the file\nline 20000 of the file\n" || a.From != 19999 { + t.Errorf("the last two lines: %q from %d, %v", a.Text, a.From, err) + } + if _, err := r.Read(p, ReadAsk{From: 30000}); err == nil || !strings.Contains(err.Error(), "20000 lines") { + t.Errorf("a line past the end: %v", err) + } +} + +// A binary file, a directory and a fifo are refused, and the fifo without waiting for a writer. +func TestOnlyATextFileIsRead(t *testing.T) { + r, dir := testReader(t) + bin := filepath.Join(dir, "bin") + write(t, bin, "ELF\x00\x01\x02") + if _, err := r.Read(bin, ReadAsk{}); err == nil || !strings.Contains(err.Error(), "binary") { + t.Errorf("a binary file: %v", err) + } + if _, err := r.Read(dir, ReadAsk{}); err == nil || !strings.Contains(err.Error(), "directory") { + t.Errorf("a directory: %v", err) + } + fifo := filepath.Join(dir, "fifo") + if err := syscall.Mkfifo(fifo, 0o644); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { _, err := r.Read(fifo, ReadAsk{}); done <- err }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "fifo") { + t.Errorf("a fifo: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a read of a fifo waits for a writer") + } +} + +// ADR 0310 §4: a line shaped like a secret is withheld, its number said, and the rest is answered. +func TestALineShapedLikeASecretIsWithheld(t *testing.T) { + r, dir := testReader(t) + p := filepath.Join(dir, "app.conf") + write(t, p, "listen = /run/app.sock\ntoken = "+aToken+"\nuser = app\n") + a, err := r.Read(p, ReadAsk{}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(a.Text, aToken) || !strings.Contains(a.Text, "user = app") || + !strings.Contains(a.Text, "listen = /run/app.sock") || fmt.Sprint(a.Withheld) != "[2]" { + t.Errorf("read: %q withheld %v", a.Text, a.Withheld) + } + m, err := r.Search(SearchAsk{Text: "token", Path: dir}) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(m) + if strings.Contains(string(body), aToken) || len(m.Matches) != 1 || len(m.Withheld) != 1 { + t.Errorf("search: %s", body) + } +} + +// The bounds of a search: matches, depth, the most asked, and time — each met is said. +func TestASearchIsBounded(t *testing.T) { + r, dir := testReader(t) + for i := 0; i < 30; i++ { + write(t, filepath.Join(dir, fmt.Sprintf("f%02d", i)), "needle\nneedle\n") + } + a, err := r.Search(SearchAsk{Text: "needle", Path: dir, Matches: 7}) + if err != nil { + t.Fatal(err) + } + if len(a.Matches) != 7 || !strings.HasPrefix(a.Bound, "matches") { + t.Errorf("seven matches asked: %d, bound %q", len(a.Matches), a.Bound) + } + if _, err := r.Search(SearchAsk{Text: "needle", Path: dir, Matches: MaxMatches + 1}); err == nil { + t.Error("more matches than the most were asked and answered") + } + if _, err := r.Search(SearchAsk{Text: "needle", Path: dir, Depth: MaxDepth + 1}); err == nil { + t.Error("deeper than the most was asked and answered") + } + deep := t.TempDir() + write(t, filepath.Join(deep, "a/b/c/shallow"), "needle\n") + write(t, filepath.Join(deep, "a/b/c/d/e/deep"), "needle\n") + a, err = r.Search(SearchAsk{Text: "needle", Path: deep, Depth: 3}) + if err != nil { + t.Fatal(err) + } + if len(a.Matches) != 1 || !strings.HasSuffix(a.Matches[0].File, "shallow") { + t.Errorf("three deep: %+v", a.Matches) + } + // Case: as asked. + write(t, filepath.Join(deep, "case"), "NEEDLE\n") + if a, _ := r.Search(SearchAsk{Text: "needle", Path: filepath.Join(deep, "case")}); len(a.Matches) != 0 { + t.Error("a match whatever the case, unasked") + } + if a, _ := r.Search(SearchAsk{Text: "needle", Path: filepath.Join(deep, "case"), CaseInsensitive: true}); len(a.Matches) != 1 { + t.Error("no match whatever the case, asked") + } + // Time: a clock past the bound stops the walk and says so. + start := time.Now() + calls := 0 + r.Now = func() time.Time { calls++; return start.Add(time.Duration(calls) * SearchTime / 3) } + a, err = r.Search(SearchAsk{Text: "needle", Path: dir}) + if err != nil || !strings.HasPrefix(a.Bound, "time") { + t.Errorf("the time bound: %q %v", a.Bound, err) + } + // A fixed string: a pattern's characters are themselves. + write(t, filepath.Join(deep, "pattern"), "a.c\nabc\n") + if a, _ := r.Search(SearchAsk{Text: "a.c", Path: filepath.Join(deep, "pattern")}); len(a.Matches) != 1 || a.Matches[0].Line != 1 { + t.Errorf("a.c as a fixed string: %+v", a.Matches) + } +} + +// The survey issue 418 was asked for: each component of a path with its owner, group, mode, whether it is a +// link and to what, and whether it is a mount point — the link not followed unless asked. +func TestStatSaysEachComponent(t *testing.T) { + r, dir := testReader(t) + data := filepath.Join(dir, "data") + write(t, filepath.Join(data, "real/file"), "x") + if err := os.Chmod(filepath.Join(data, "real"), os.ModeSetgid|0o750); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real", filepath.Join(data, "link")); err != nil { + t.Fatal(err) + } + a, err := r.Stat([]string{filepath.Join(data, "link"), filepath.Join(data, "real/file")}, false) + if err != nil { + t.Fatal(err) + } + link := a.Paths[0] + last := link.Components[len(link.Components)-1] + if link.Components[0].Path != "/" || last.Type != "link" || last.Link != "real" || link.LeadsTo != "" { + t.Errorf("the link, not followed: %+v", link) + } + var dataC, realC Component + for _, c := range a.Paths[1].Components { + switch c.Path { + case data: + dataC = c + case filepath.Join(data, "real"): + realC = c + } + } + if dataC.Mount == nil || dataC.Mount.FS != "nfs4" { + t.Errorf("the mount point is not said: %+v", dataC) + } + if realC.Mode != "2750" || realC.Type != "directory" || realC.UID == nil || realC.Perms == "" { + t.Errorf("a component's mode: %+v", realC) + } + if uid := *realC.UID; uid == 1000 && realC.Owner != "ops" { + t.Errorf("owner %d named %q", uid, realC.Owner) + } + a, err = r.Stat([]string{filepath.Join(data, "link")}, true) + if err != nil { + t.Fatal(err) + } + if a.Paths[0].LeadsTo != filepath.Join(data, "real") || a.Paths[0].Leads == nil || a.Paths[0].Leads.Type != "directory" { + t.Errorf("followed: %+v", a.Paths[0]) + } + a, err = r.Stat([]string{filepath.Join(dir, "missing/deeper")}, false) + if err != nil || !strings.Contains(a.Paths[0].Stopped, "does not exist") { + t.Errorf("a missing path: %+v %v", a, err) + } + if _, err := r.Stat([]string{"relative/path"}, false); err == nil { + t.Error("a relative path was answered") + } + many := make([]string, MaxPaths+1) + for i := range many { + many[i] = "/" + } + if _, err := r.Stat(many, false); err == nil { + t.Error("more paths than the most were answered") + } +} + +func TestReadlinkSaysTheTargetAndWhereItLeads(t *testing.T) { + r, dir := testReader(t) + write(t, filepath.Join(dir, "target"), "") + if err := os.Symlink("target", filepath.Join(dir, "l1")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("l1", filepath.Join(dir, "l2")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("nowhere", filepath.Join(dir, "dangling")); err != nil { + t.Fatal(err) + } + a, err := r.Readlink(filepath.Join(dir, "l2")) + if err != nil || !a.IsLink || a.Link != "l1" || a.LeadsTo != filepath.Join(dir, "target") || !a.Exists { + t.Errorf("l2: %+v %v", a, err) + } + a, err = r.Readlink(filepath.Join(dir, "dangling")) + if err != nil || !a.IsLink || a.Exists || !strings.Contains(a.Stopped, "does not exist") { + t.Errorf("dangling: %+v %v", a, err) + } +} + +func TestListSaysEveryEntry(t *testing.T) { + r, dir := testReader(t) + write(t, filepath.Join(dir, "d/a"), "abc") + if err := os.Symlink("a", filepath.Join(dir, "d/b")); err != nil { + t.Fatal(err) + } + a, err := r.List(filepath.Join(dir, "d")) + if err != nil { + t.Fatal(err) + } + if a.Count != 2 || a.Entries[0].Name != "a" || *a.Entries[0].Size != 3 || a.Entries[1].Type != "link" || a.Entries[1].Link != "a" { + t.Errorf("list: %+v", a) + } + for i := 0; i < MaxEntries+5; i++ { + write(t, filepath.Join(dir, "many", fmt.Sprintf("e%05d", i)), "") + } + a, err = r.List(filepath.Join(dir, "many")) + if err != nil || len(a.Entries) != MaxEntries || a.Count != MaxEntries+5 || a.Cut == "" { + t.Errorf("a long directory: %d of %d, cut %q, %v", len(a.Entries), a.Count, a.Cut, err) + } +} + +// getent passwd, getent group and id: an account with every group it is in, a group with its members, never a +// password field. +func TestAccountAndGroupLookup(t *testing.T) { + r, _ := testReader(t) + a, err := r.Account("ops", "") + if err != nil { + t.Fatal(err) + } + u := a.Accounts[0] + sort.Strings(u.Groups) + if u.UID != 1000 || u.Group != "ops" || u.Shell != "/usr/bin/zsh" || fmt.Sprint(u.Groups) != "[docker ops wheel]" { + t.Errorf("ops: %+v", u) + } + if a, err := r.Account("961", ""); err != nil || a.Accounts[0].Name != "mesh-reads" { + t.Errorf("by number: %+v %v", a, err) + } + g, err := r.Account("", "docker") + if err != nil || g.Groups[0].GID != 970 || fmt.Sprint(g.Groups[0].Members) != "[ops builder]" { + t.Errorf("docker: %+v %v", g, err) + } + if _, err := r.Account("nobody-here", ""); err == nil { + t.Error("a missing account was answered") + } + all, err := r.Account("", "") + if err != nil || len(all.Accounts) != 3 || len(all.Groups) != 5 { + t.Errorf("all: %+v %v", all, err) + } + body, _ := json.Marshal(all) + if strings.Contains(string(body), `"x"`) || strings.Contains(string(body), "password") { + t.Errorf("a password field is answered: %s", body) + } +} + +// No answer is larger than the bound, whatever a verb made. +func TestNoAnswerIsLargerThanTheBound(t *testing.T) { + if _, err := bounded(strings.Repeat("x", MaxAnswer+1), nil); err == nil { + t.Error("an answer over the bound was given") + } + if _, err := bounded("small", nil); err != nil { + t.Error(err) + } +} + +// The seat's six verbs, as the controller defines them (mesh-controller internal/catalogue/reads.go). +func TestItServesTheSeatsVerbs(t *testing.T) { + r, _ := testReader(t) + var names []string + for _, tool := range tools(r) { + names = append(names, tool.Name) + } + if fmt.Sprint(names) != "[node-reads.stat node-reads.readlink node-reads.list node-reads.read node-reads.search node-reads.account]" { + t.Errorf("verbs: %v", names) + } +} + +// The review of #244 (required 1): a link flipped between a harmless file and a secret place, while it is read +// and searched, never answers the secret place's content — what was opened is judged, not the name. +func TestALinkFlippedDuringAReadNeverReadsASecretPlace(t *testing.T) { + r, dir := testReader(t) + secret := filepath.Join(dir, "var/lib/mesh-reads") + r.Places = []string{secret} + write(t, filepath.Join(secret, "broker"), "THE-BROKER-CREDENTIAL\n") + write(t, filepath.Join(dir, "plain"), "THE-PLAIN-WORDS\n") + link := filepath.Join(dir, "shared/link") + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + t.Fatal(err) + } + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + targets := []string{filepath.Join(dir, "plain"), filepath.Join(secret, "broker")} + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + tmp := link + ".new" + os.Remove(tmp) + if os.Symlink(targets[i%2], tmp) == nil { + os.Rename(tmp, link) + } + } + }() + leaked := 0 + for i := 0; i < 5000; i++ { + if a, err := r.Read(link, ReadAsk{}); err == nil && strings.Contains(a.Text, "BROKER") { + leaked++ + } + if a, err := r.Search(SearchAsk{Text: "BROKER", Path: filepath.Dir(link)}); err == nil && len(a.Matches) > 0 { + leaked++ + } + } + close(stop) + <-done + if leaked > 0 { + t.Errorf("a secret place was answered %d times through a flipped link", leaked) + } +} + +// The review of #244 (required 2): every line of a private key's block is withheld, in a read and in a search, +// though the file's name is not refused. +func TestEveryLineOfAKeyBlockIsWithheld(t *testing.T) { + r, dir := testReader(t) + key, err := rsa.GenerateKey(crand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + body := string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) + p := filepath.Join(dir, "server.pem") + write(t, p, "# the server's key\n"+body+"after = the block\n") + a, err := r.Read(p, ReadAsk{}) + if err != nil { + t.Fatal(err) + } + lines := strings.Count(body, "\n") + for _, l := range strings.Split(strings.TrimSpace(body), "\n")[1:] { + if strings.Contains(a.Text, l) && !strings.HasPrefix(l, "-----END") { + t.Fatalf("a line of the key was answered: %q", l) + } + } + if len(a.Withheld) != lines || !strings.Contains(a.Text, "after = the block") || !strings.Contains(a.Text, "the server's key") { + t.Errorf("withheld %d of the block's %d lines; text:\n%s", len(a.Withheld), lines, a.Text) + } + // A search for a character every base64 line holds answers none of the key. + m, err := r.Search(SearchAsk{Text: "A", Path: p, Matches: MaxMatches}) + if err != nil { + t.Fatal(err) + } + for _, x := range m.Matches { + if !strings.HasPrefix(x.Text, "‹withheld") { + t.Errorf("a search answered a line of the key: %q", x.Text) + } + } +} + +// The review of #244 (required 3 and should): histories, browser profiles, a process's private entries and the +// cluster credentials are refused by name. +func TestHistoriesProfilesAndProcessesAreRefusedByName(t *testing.T) { + var r Refuser + for _, p := range []string{"/home/ops/.zsh_history", "/home/ops/.bash_history", "/root/.psql_history", + "/home/ops/.viminfo", "/home/ops/.mozilla/firefox/x.default/logins.json", + "/home/ops/.config/google-chrome/Default/Cookies", "/home/ops/.config/chromium/Default/Login Data", + "/proc/self/environ", "/proc/1234/mem", "/proc/self/fd/3", "/proc/thread-self/root/etc/shadow", + "/etc/rancher/k3s/k3s.yaml", "/etc/kubernetes/admin.conf"} { + if r.Refused(p) == "" { + t.Errorf("%s is not refused", p) + } + } + for _, p := range []string{"/proc/mdstat", "/proc/self/status", "/home/ops/.config/i3/config", "/etc/hosts"} { + if why := r.Refused(p); why != "" { + t.Errorf("%s is refused: %s", p, why) + } + } +} diff --git a/modules/mesh-reads/cmd/mesh-reads/refuse.go b/modules/mesh-reads/cmd/mesh-reads/refuse.go new file mode 100644 index 00000000..9a132046 --- /dev/null +++ b/modules/mesh-reads/cmd/mesh-reads/refuse.go @@ -0,0 +1,129 @@ +package main + +import ( + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/outward" +) + +// What is refused by name, before any read (novox/hq ADR 0310 §5), and what an answer withholds (§4). +// +// **Two lists of places.** The places the mesh knows a secret is kept on this node — every module's own-secret +// directory, every credential file and grant directory, the vault's directories — come from the controller, +// which composes them from the manifests (MESH_SECRET_PLACES). The known credential files of the programs +// on a machine are this module's list, below: the mesh does not write them, so nothing else names them. Both +// hold whatever the account's file modes would allow: the file modes are the first bound, the names the second. + +// knownDirs are directories that hold nothing but credentials, wherever they are: an account's keys. +var knownDirs = map[string]bool{ + ".ssh": true, ".gnupg": true, ".password-store": true, ".aws": true, ".kube": true, ".azure": true, + ".mozilla": true, ".thunderbird": true, // browser and mail profiles: saved logins and cookies +} + +// knownNames are file names that are a credential whatever directory they are in. +var knownNames = map[string]bool{ + ".credentials.json": true, // the agent's credentials (Claude Code) + ".netrc": true, ".pgpass": true, ".git-credentials": true, ".npmrc": true, ".pypirc": true, + ".my.cnf": true, ".vault-token": true, "credentials": true, "credentials.json": true, + "id_rsa": true, "id_dsa": true, "id_ecdsa": true, "id_ed25519": true, "id_ecdsa_sk": true, "id_ed25519_sk": true, + "shadow": true, "gshadow": true, "shadow-": true, "gshadow-": true, "opasswd": true, + "broker": true, // the mesh's name for a module's bus credential + // Shell and program histories: where a token is typed. + ".bash_history": true, ".zsh_history": true, ".histfile": true, ".python_history": true, ".psql_history": true, + ".mysql_history": true, ".lesshst": true, ".viminfo": true, ".node_repl_history": true, ".sqlite_history": true, + ".rediscli_history": true, +} + +// knownSuffixes are file name endings a credential has. +var knownSuffixes = []string{".key", ".p12", ".pfx", ".jks", ".keystore", ".creds", ".nk", ".kdbx", "_key"} + +// knownPaths are the places on a machine that hold credentials and are not a module's. +var knownPaths = []string{ + "/etc/ssl/private", "/etc/wireguard", "/etc/NetworkManager/system-connections", "/etc/ipsec.d/private", + "/var/lib/iwd", "/etc/docker/key.json", "/root/.docker/config.json", "/var/lib/sss/secrets", + "/etc/rancher", "/etc/kubernetes/admin.conf", +} + +// knownProfiles are directories under an account's .config that hold a browser's saved logins and cookies. +var knownProfiles = map[string]bool{"google-chrome": true, "chromium": true, "BraveSoftware": true, "vivaldi": true, + "microsoft-edge": true} + +// procPrivate are the entries of a process under /proc that reach its memory, its environment or its files. +var procPrivate = map[string]bool{"environ": true, "mem": true, "fd": true, "map_files": true, "root": true, + "cwd": true} + +// Refuser says whether a path is a place a secret is kept. +type Refuser struct { + // Places are the mesh's, from the controller: refused with everything beneath them. + Places []string +} + +// Refused says why a path may not be read, or "" when it may. The path is clean and absolute. +func (r Refuser) Refused(path string) string { + path = filepath.Clean(path) + for _, p := range r.Places { + if within(path, p) { + return "it is a place the mesh keeps a secret (" + p + ")" + } + } + for _, p := range knownPaths { + if within(path, p) { + return "it is a place a machine keeps credentials (" + p + ")" + } + } + parts := strings.Split(strings.TrimPrefix(path, "/"), "/") + if len(parts) >= 3 && parts[0] == "proc" && procPrivate[parts[2]] { + return "it reaches a process's " + parts[2] + } + for i, part := range parts { + if i > 0 && parts[i-1] == ".config" && knownProfiles[part] { + return "it is " + part + "'s profile, which keeps saved logins" + } + if knownDirs[part] { + return "it is in " + part + ", which holds an account's credentials" + } + if i == len(parts)-1 { + if knownNames[part] { + return part + " is a known credential file" + } + for _, s := range knownSuffixes { + if strings.HasSuffix(part, s) && len(part) > len(s) { + return part + " is named as a key or a credential (" + s + ")" + } + } + if strings.HasSuffix(part, ".pem") && (strings.Contains(part, "key") || strings.Contains(part, "private")) { + return part + " is named as a private key" + } + if part == "config.json" && i > 0 && parts[i-1] == ".docker" { + return "a container registry login is kept there" + } + } + } + return "" +} + +func within(path, place string) bool { + return path == place || strings.HasPrefix(path, strings.TrimSuffix(place, "/")+"/") +} + +// Withheld is one piece of an answer the content rule kept back: where it was, and the class it carried. +type Withheld struct { + At string `json:"at"` + Class string `json:"why"` +} + +// clean is the content rule's secret class (mesh-sdk outward, novox/hq ADR 0310 §4) over one piece of an +// answer: the piece, or a mark in its place when it is shaped like a secret. +// +// **What a file holds, not what the file system names.** It judges a file's lines, a matching line, a link's +// target, a mount's source and an account's words. A path and an entry's name are not judged: they are what +// the caller named or the file system calls a thing, and the rule's random-run test takes a temporary +// directory's or a container layer's name for a token, which would withhold the very survey asked for. A +// place where a secret sits by name is refused before any read instead (Refuser). +func clean(s string) (string, string) { + if refusal, ok := outward.CheckSecret(s); !ok { + return "‹withheld: " + refusal.What + "›", refusal.What + } + return s, "" +} diff --git a/modules/mesh-reads/go.mod b/modules/mesh-reads/go.mod new file mode 100644 index 00000000..25037a27 --- /dev/null +++ b/modules/mesh-reads/go.mod @@ -0,0 +1,5 @@ +module mesh-reads + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2 diff --git a/modules/mesh-reads/go.sum b/modules/mesh-reads/go.sum new file mode 100644 index 00000000..4367b997 --- /dev/null +++ b/modules/mesh-reads/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2 h1:1E5iapAO1o4vkwV/8CUBQvNfL3OjXRtiQGBaKHXJVW8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/mesh-reads/module.json b/modules/mesh-reads/module.json new file mode 100644 index 00000000..4a664485 --- /dev/null +++ b/modules/mesh-reads/module.json @@ -0,0 +1,54 @@ +{ + "module": "mesh-reads", + "version": "1", + "runs-as": "mesh-reads", + "claims": [ + { + "name": "node-reads", + "scope": "node", + "serves": [ + "stat", + "readlink", + "list", + "read", + "search", + "account" + ] + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-reads", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-reads", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-reads" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-reads", + "place": "." + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-reads", + "binary": "mesh-reads", + "loads": [ + "mesh-reads" + ] + } + ] + } +}