diff --git a/merge-check-typescript.sh b/merge-check-typescript.sh index 3b0ed856..b32bb08f 100644 --- a/merge-check-typescript.sh +++ b/merge-check-typescript.sh @@ -23,9 +23,51 @@ # # By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads # /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same. +# +# **In an agent's session, outside the toolchain image** (novox/hq issue 742), the same node_modules is +# assembled first, in a throwaway directory under the check's own workspace, the way the image's `deps` stage +# assembles it: the toolchain's own package.json and lock (mesh-tools' node-tools/, read from the forge this +# clone came from, so the list is never copied here) installed with `npm ci`, then the SDK the mesh last +# published (`@novox/mesh-sdk@latest`), the @novox scope from the forge's package registry, read without a +# credential. Node and npm are the dev-tools module's; nothing is installed on the machine, no npm +# configuration or cache of the account's is read or written. Only when neither TOOLCHAIN_MODULES nor +# /app/node_modules is there: the forge's check, in the image, fetches nothing, as before. The forge is +# MESH_FORGE (default: the origin remote's), the owner MESH_FORGE_OWNER (default: the origin's owner). set -eu +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + deps="${TOOLCHAIN_MODULES:-/app/node_modules}" +if [ -z "${TOOLCHAIN_MODULES:-}" ] && [ ! -f "$deps/typescript/bin/tsc" ]; then + origin=$(git remote get-url origin 2>/dev/null || true) + forge="${MESH_FORGE:-$(printf '%s\n' "$origin" | sed -n 's#^\(https\{0,1\}://[^/]*\)/.*#\1#p')}" + owner="${MESH_FORGE_OWNER:-$(printf '%s\n' "$origin" | sed -n 's#^https\{0,1\}://[^/]*/\([^/]*\)/.*#\1#p')}" + if [ -z "$forge" ] || [ -z "$owner" ]; then + echo "NOT CHECKED: no TypeScript toolchain at $deps, and no forge to assemble one from (origin is '$origin'; set MESH_FORGE and MESH_FORGE_OWNER, or TOOLCHAIN_MODULES)" + exit 1 + fi + echo "no TypeScript toolchain at $deps: assembling one from $forge/$owner/mesh-tools node-tools/ and $owner's package registry (novox/hq issue 742)" + tc="$work/toolchain" + mkdir -p "$tc" + printf '@%s:registry=%s/api/packages/%s/npm/\n' "$owner" "$forge" "$owner" >"$work/npmrc" + # Read with Node's own fetch (dev-tools declares Node, not curl); a status that is not 2xx fails. + if ! { node -e ' + const [base, dir] = process.argv.slice(1); + Promise.all(["package.json", "package-lock.json"].map(async (f) => { + const r = await fetch(base + f); + if (!r.ok) throw new Error(base + f + ": " + r.status); + require("fs").writeFileSync(dir + "/" + f, await r.text()); + })).catch((e) => { console.error(e.message); process.exit(1); });' \ + "$forge/$owner/mesh-tools/raw/branch/main/node-tools/" "$tc" && + (cd "$tc" && export npm_config_userconfig="$work/npmrc" npm_config_cache="$work/npm-cache" && + npm ci --ignore-scripts --no-audit --no-fund --loglevel=error && + npm install --ignore-scripts --no-audit --no-fund --no-save --loglevel=error "@$owner/mesh-sdk@latest"); }; then + echo "NOT CHECKED: the TypeScript toolchain could not be assembled from $forge (novox/hq issue 742)" + exit 1 + fi + deps="$tc/node_modules" +fi tsc="$deps/typescript/bin/tsc" if [ ! -f "$tsc" ]; then echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain" @@ -39,8 +81,6 @@ touched=$({ printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p' } | sed '/^$/d' | sort -u) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT failed="" for m in $touched; do d="modules/$m" diff --git a/modules/build-agent/Dockerfile b/modules/build-agent/Dockerfile index 8abeaa19..55f388d9 100644 --- a/modules/build-agent/Dockerfile +++ b/modules/build-agent/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /mesh-builder ./cmd/mesh-builder +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -buildvcs=false -ldflags=-buildid= -o /mesh-builder ./cmd/mesh-builder # Unlike mesh-controller's own FROM scratch (ADR 0006: nothing to audit but one binary), the build # machine's whole job is shelling out to git and docker — it needs a real userland to do that in, diff --git a/modules/gitea/README.md b/modules/gitea/README.md index cb5edb23..c769a732 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -52,7 +52,10 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag | `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path | **Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266): -never the repository's default branch, never a branch a protection rule names. A rule covers a branch as the +never the repository's default branch, never a branch a protection rule names. The tools' account may not read the +rules themselves (the forge keeps them for a repository's administrator, novox/hq ADR 0365): for it an existing branch +is judged by the protection the forge says it applies there, and a branch not made yet by the forge itself, which +refuses the commit that would make a branch a rule guards. A rule covers a branch as the forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob (`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the @@ -66,7 +69,7 @@ the commit the branch (or, for a new branch, its base) was read at, never at its forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the one the forge says it made the commit on; when the branch moved between the read and the commit, the answer -says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the +says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the tools' account, so its sha is not the local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a branch not at the commit the forge named is a failure, never a success. @@ -75,8 +78,10 @@ is read once and searched in memory; a binary file is passed over, a file over 4 `skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`. -It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), and one -tool call takes at most 50 seconds. +It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the +tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and +keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool +call takes at most 50 seconds. `go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on a new branch from its base; an existing branch committed on; the default branch, a protected branch and unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 55bba9dc..0782397a 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -8,6 +8,7 @@ import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js"; import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js"; import { isCommit } from "./checks.js"; +import { TOOLS_ACCOUNT } from "./tools-account.js"; /** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every * call, whether the names came from a caller or from the forge's own answer. */ @@ -124,10 +125,12 @@ export interface ReadyMark { /** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which * the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors= …`. A file - * that cannot be read leaves the mesh's own forge account alone, said; never an empty list read as everyone. */ + * that cannot be read leaves the mesh's own forge accounts alone, said; never an empty list read as everyone: the + * account the tools act as (novox/hq ADR 0365) and the admin account, whose marks were made before it. */ export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER, - read: (f: string) => string = (f) => readFileSync(f, "utf8")): string[] { - const fallback = own ? [own] : []; + read: (f: string) => string = (f) => readFileSync(f, "utf8"), + tools = process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT): string[] { + const fallback = [...new Set([tools, own].filter((a): a is string => !!a))]; if (!file) return fallback; try { const line = read(file).split("\n").find((l) => l.startsWith("authors=")); @@ -444,13 +447,15 @@ export class GiteaClient { * fault in the announcer cannot hide itself. Reads only. */ async openPulls(readyAuthors: string[] = []): Promise { const out: OpenPull[] = []; - const rules = new Map(); + const rules = new Map(); for (const found of await this.searchOpenPulls()) { const full = `${found.owner}/${found.name}`; const pull = await this.getPullRequest(found.owner, found.name, found.number); if (pull.state !== "open" || !pull.head_sha) continue; - if (!rules.has(full)) rules.set(full, await this.branchProtections(found.owner, found.name).catch(() => undefined)); - const rule = rules.get(full)?.find((p) => (p.rule_name ?? p.branch_name) === (pull.base || "main")); + // The base's protection as the forge applies it to that branch (appliedProtection): readable without administration. + const base = `${full}:${pull.base || "main"}`; + if (!rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base || "main").catch(() => undefined)); + const rule = rules.get(base); const checks = await this.headChecks(found.owner, found.name, pull.head_sha); const labels = found.labels ?? []; // The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label. @@ -673,16 +678,44 @@ export class GiteaClient { // ---- Branch protection ---- - /** A repository's branch protection rules. */ + /** A repository's branch protection rules. The forge answers them to a repository's administrator alone, which the + * tools' account is not (novox/hq ADR 0365 §2): for it this is refused (403), and what a branch's protection asks + * is read with appliedProtection. */ async branchProtections(owner: string, repo: string): Promise { return (await this.request(`${repoPath(owner, repo)}/branch_protections`)) ?? []; } - /** The rule named for a branch, null when it has none. */ + /** The rule named for a branch, null when it has none — from the rules' list, so a repository administrator's. */ async branchProtection(owner: string, repo: string, rule: string): Promise { return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null; } + /** + * The protection that covers an existing branch, as the forge applies it, or null when none does. Read from the + * branch itself, which anyone who reads the code may read — the rules' own list is a repository administrator's + * (novox/hq ADR 0365 §2) — and as the forge matches it, a glob rule included (its name is the rule's). The + * branch's `user_can_push` stands for `enable_push`: whether this account may push there. A branch that does not + * exist is null: nothing covers it yet, and the forge judges a push that makes it. + */ + async appliedProtection(owner: string, repo: string, branch: string): Promise { + let b: any; + try { + b = await this.request(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`); + } catch (err) { + if (/: 404 /.test(String((err as Error)?.message))) return null; + throw err; + } + if (!b?.protected) return null; + return { + rule_name: String(b.effective_branch_protection_name || branch), + branch_name: branch, + enable_push: Boolean(b.user_can_push), + enable_status_check: Boolean(b.enable_status_check), + status_check_contexts: Array.isArray(b.status_check_contexts) ? b.status_check_contexts.map(String) : [], + required_approvals: Number(b.required_approvals ?? 0), + }; + } + /** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the * branch is edited — its required statuses replaced by these, everything else it says kept unless * asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */ @@ -914,12 +947,26 @@ export class GiteaAdmin { return { status: res.status, body }; } + /** One admin-API call with a JSON body, answering the status and body (the system webhook, novox/hq issue 729). */ + call(method: string, path: string, body?: unknown): Promise { + return this.request(path, { method, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + } + /** Fail with the forge's own message when a status the caller did not expect comes back. */ private static fail(path: string, res: AdminResponse): never { const detail = typeof res.body === "string" ? res.body : JSON.stringify(res.body); throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`); } + /** A repository's branch protection rules, read with the administrator account — the forge answers them to a + * repository's administrator alone, which the tools' account is not (novox/hq ADR 0365 §2, ADR 0375). Only a read. */ + async branchProtections(owner: string, repo: string): Promise { + const path = `${repoPath(owner, repo)}/branch_protections`; + const res = await this.request(path); + if (res.status !== 200) GiteaAdmin.fail(path, res); + return Array.isArray(res.body) ? res.body : []; + } + /** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent * create won the race) are all success. */ async ensureOrg(name: string): Promise { @@ -943,12 +990,16 @@ export class GiteaAdmin { // unit the team does not have — so code read must be said here: without it gitea answers a // member's clone of a private repository with "not found", which is how the builder's first // credentialed clone failed against a team that named only packages. - const units = { + return this.ensureTeamWith(org, team, { permission: "read", units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" }, includes_all_repositories: true, can_create_org_repo: false, - }; + }); + } + + /** Ensure a team exists with exactly these units, and return its id (ensureTeam; the tools' team, tools-account.ts). */ + async ensureTeamWith(org: string, team: string, units: Record): Promise { const found = await this.findTeam(org, team); if (found !== null) { const patch = await this.request(`/teams/${found}`, { @@ -983,7 +1034,7 @@ export class GiteaAdmin { * The edit path is taken only when the user actually exists. A 422 from the create is also what * a plain validation failure returns, and reading it as "already there" made the follow-up edit * 404 — burying the create's own message, which is the one that says what is actually wrong. */ - async ensureUser(username: string, password: string, email: string): Promise { + async ensureUser(username: string, password: string, email: string, plain = false): Promise { const res = await this.request("/admin/users", { method: "POST", body: JSON.stringify({ username, email, password, must_change_password: false }), @@ -997,7 +1048,9 @@ export class GiteaAdmin { // login_name is required by the admin edit endpoint; for a local user it is the username. // active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong // password, so the provisioner's check reports it lost; applying again must undo both. - body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false }), + // plain: never a site administrator — one made so by hand is made a plain account again (novox/hq ADR 0365). + body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false, + ...(plain ? { admin: false } : {}) }), }); if (patch.status === 200) return; GiteaAdmin.fail(`/admin/users/${username}`, patch); diff --git a/modules/gitea/cmd/forge/account_test.go b/modules/gitea/cmd/forge/account_test.go new file mode 100644 index 00000000..d37c6594 --- /dev/null +++ b/modules/gitea/cmd/forge/account_test.go @@ -0,0 +1,120 @@ +package main + +// The commit and the reads act as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the module +// keeps for `mesh-tools`, never the admin login. That account is no repository's administrator, so the forge +// refuses it the protection rules (403, as gitea's reqAdmin does); the commit then judges an existing branch by the +// protection the forge says it applies, and leaves a branch not made yet to the forge's own judgement. + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "gitea/internal/forgehttp" +) + +const toolsToken = "tools-token-never-shown" + +// asTools serves the fake forge to a bundle holding the tools' token: only that token is taken, the rules are +// refused as to a non-administrator, and the branches named protected say so as the forge does. +func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes string) (*Forge, *[]string) { + dir := t.TempDir() + file := filepath.Join(dir, "token") + if err := os.WriteFile(file, []byte(toolsToken+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file+".scopes", []byte(scopes+"\n"), 0o600); err != nil { + t.Fatal(err) + } + var seen []string + const repo = "/api/v1/repos/acme/app" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Method+" "+r.URL.Path) + if _, _, basic := r.BasicAuth(); basic || r.Header.Get("Authorization") != "token "+toolsToken { + http.Error(w, "unauthorized", 401) + return + } + if r.URL.Path == repo+"/branch_protections" { + http.Error(w, `{"message":"token does not have the administration of this repository"}`, 403) + return + } + if name := strings.TrimPrefix(r.URL.Path, repo+"/branches/"); r.Method == "GET" && name != r.URL.Path { + if rule, ok := protected[name]; ok { + _, _ = w.Write([]byte(`{"name":"` + name + `","commit":{"id":"c0"},"protected":true,"effective_branch_protection_name":"` + rule + `"}`)) + return + } + } + r.SetBasicAuth("mesh-admin", f.password) // the fake's own door, past the token checked above + f.handle(w, r) + })) + t.Cleanup(srv.Close) + return &Forge{Client: forgehttp.Client{URL: srv.URL, TokenFile: file, Account: "mesh-tools"}}, &seen +} + +func TestTheToolsAccountCommitsOnANewBranchWithItsTokenThoughTheRulesAreNotItsToRead(t *testing.T) { + f := newFake(t) + forge, _ := asTools(t, f, nil, "read:user,write:issue,write:repository for mesh-tools") + got, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err != nil { + t.Fatal(err) + } + if len(f.writes) != 1 || !got.Created { + t.Fatalf("%d writes, %+v", len(f.writes), got) + } +} + +func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefaultBranch(t *testing.T) { + for branch, rule := range map[string]string{"release/1.0": "release/*", "fix/1-old": "fix/1-old", "main": ""} { + f := newFake(t) + protected := map[string]string{} + if rule != "" { + protected[branch] = rule + } + forge, _ := asTools(t, f, protected, "read:user,write:issue,write:repository for mesh-tools") + _, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": branch, + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") { + t.Errorf("%s: %v", branch, err) + } + if rule != "" && !strings.Contains(err.Error(), rule) { + t.Errorf("%s: the rule %s is not named: %v", branch, rule, err) + } + if len(f.writes)+len(f.patches) != 0 { + t.Errorf("%s: something was written", branch) + } + } +} + +func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) { + f := newFake(t) + // What the module kept before ADR 0365: the admin's token, its scopes line naming no account. + forge, seen := asTools(t, f, nil, "read:user,write:issue,write:repository") + _, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "not mesh-tools's") { + t.Fatalf("%v", err) + } + if len(*seen) != 0 { + t.Fatalf("the forge was asked %v", *seen) + } +} + +func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) { + t.Setenv("MESH_GITEA_URL", "http://127.0.0.1:3000") + t.Setenv("MESH_GITEA_TOKEN_FILE", "/state/token") + t.Setenv("MESH_GITEA_TOOLS_USER", "") + t.Setenv("MESH_GITEA_ADMIN_USER", "") + t.Setenv("MESH_GITEA_ADMIN_PASSWORD_FILE", "") + f, err := ForgeFromEnv() + if err != nil { + t.Fatal(err) + } + if f.TokenFile != "/state/token" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" { + t.Fatalf("%+v", f.Client) + } +} diff --git a/modules/gitea/cmd/forge/commit.go b/modules/gitea/cmd/forge/commit.go index 6976da57..d3c2af52 100644 --- a/modules/gitea/cmd/forge/commit.go +++ b/modules/gitea/cmd/forge/commit.go @@ -9,6 +9,8 @@ import ( "net/url" "strings" "unicode/utf8" + + "gitea/internal/forgehttp" ) // gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull @@ -211,7 +213,32 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { // Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed). var rules []Protection if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil { - return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) + var status *forgehttp.StatusError + if !errors.As(err, &status) || status.Status != http.StatusForbidden { + return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) + } + // The rules are a repository administrator's to read, and the tools' account is none (novox/hq ADR 0365 + // §2). What it may read is the protection the forge applies to an existing branch: one protected is + // refused here. A branch not made yet is covered by no rule this account can see, and the forge itself + // judges the commit that makes it against every rule, refusing one a rule guards. + if err := targetRefused(a.Branch, r.DefaultBranch, nil); err != nil { + return nil, err + } + b, err := f.branch(ctx, repo, a.Branch) + if err != nil { + return nil, fmt.Errorf("whether a protection rule guards %s of %s/%s could not be read, so nothing was sent: %w", a.Branch, a.Owner, a.Repo, err) + } + if b != nil && b.Protected { + rule := b.Rule + if rule == "" { + rule = a.Branch + } + rules = []Protection{{RuleName: rule}} + if !ruleCovers(rule, a.Branch) { + return nil, refuse("a commit on %s is refused: the protection rule %s guards it (novox/hq ADR 0266). Put it on a "+ + "branch of its own. Nothing was sent", a.Branch, rule) + } + } } if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil { return nil, err diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go index 3637116d..a28a915b 100644 --- a/modules/gitea/cmd/forge/forge.go +++ b/modules/gitea/cmd/forge/forge.go @@ -11,8 +11,8 @@ import ( "gitea/internal/forgehttp" ) -// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account -// (internal/forgehttp, shared with the npm-registry bundle beside it). +// The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq +// ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it). // Forge is the forge's HTTP interface. type Forge struct { @@ -91,12 +91,15 @@ func (p Protection) name() string { return p.BranchName } -// Branch is one branch with the commit it points at. +// Branch is one branch with the commit it points at, and the protection the forge applies to it: what an account +// that may not read the rules themselves (the tools' account, novox/hq ADR 0365) knows of them. type Branch struct { Name string `json:"name"` Commit struct { ID string `json:"id"` } `json:"commit"` + Protected bool `json:"protected"` + Rule string `json:"effective_branch_protection_name"` } // branch is the named branch, nil when the forge has none of that name. diff --git a/modules/gitea/cmd/forge/forge_test.go b/modules/gitea/cmd/forge/forge_test.go index 7d9464df..1d315cc3 100644 --- a/modules/gitea/cmd/forge/forge_test.go +++ b/modules/gitea/cmd/forge/forge_test.go @@ -619,11 +619,17 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { if len(a.Loads) != 1 || a.Loads[0] != a.Binary { t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary) } - for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { + // The tools' own account, never the admin login (novox/hq ADR 0365 §2). + for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"} { if a.Env[k] == "" { t.Errorf("the bundle is not given %s", k) } } + for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { + if a.Env[k] != "" { + t.Errorf("the bundle is given %s: it acts as the tools' account, not the administrator", k) + } + } } } if !found { diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index d64045a7..c730bc2a 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -40,10 +40,11 @@ import { emit, on } from "@novox/mesh-sdk/events"; import { registerModuleTools } from "@novox/mesh-sdk/tools"; import { commitStatusTool } from "./delivery.js"; import { mergeTool } from "./merge.js"; -import { GiteaClient, movedSince } from "./client.js"; +import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, - type RepoCheckFacts, reposToLook } from "./pulls.js"; + repoCheckFacts, reposToLook } from "./pulls.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; +import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is @@ -145,57 +146,9 @@ async function pollMerged(client: GiteaClient): Promise { // at once. const fresh = !!pull.merged_at && !!since && pull.merged_at > since; if (primedMerges && fresh) { - // What it changed, asked for only now: a module is rebuilt because a file inside its own - // directory moved, and without this every module built from a repository is rebuilt for a - // change to any of them (novox/hq 04-ISSUES/131). - const changed = await client.listPullFiles(repo.owner, repo.name, pull.number); - // Which of the directories they are in hold a module at the merge commit (novox/hq issue 278): a - // change inside one is that module's, held or not, and only a file in none is shared code. Not - // said when the list is cut or the forge could not be asked; the mesh then keeps its old rule. - let moduleDirs: string[] | null = null; - if (!changed.truncated) { - try { - moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, pull.merge_commit_sha, changed.paths); - } catch (err) { - console.error(`[gitea] ${repo.full_name}#${pull.number}: which directories hold a module could not be read, ` + - `so the mesh reads its files by the old rule — ${err instanceof Error ? err.message : String(err)}`); - } - } - // The head it merged, and what its head was checked as (novox/hq ADR 0239): the delivery it was, made - // from the forge's word when its owner never heard the head. - let headChecks: Record | null = null; - if (pull.head_sha) { - try { - headChecks = await client.commitStatuses(repo.owner, repo.name, pull.head_sha); - } catch (err) { - console.error(`[gitea] ${repo.full_name}#${pull.number}: its head's statuses could not be read — ${err instanceof Error ? err.message : err}`); - } - } - await emit("pull.merged", { - owner: repo.owner, - repo: repo.name, - number: pull.number, - title: pull.title, - body: pull.body, - head_sha: pull.head_sha, - ...(headChecks ? { head_checks: headChecks } : {}), - head: pull.head, - base: pull.base, - merge_commit_sha: pull.merge_commit_sha, - merged_at: pull.merged_at, - clone_url: repo.clone_url, - html_url: pull.html_url, - paths: changed.paths, - paths_truncated: changed.truncated, - // Which of them the merge deleted (novox/hq ADR 0236): a module whose manifest went is forgotten, - // not built. - removed: changed.removed, - ...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}), - }); - // Said, because a trigger that fires silently is indistinguishable from one that did not - // fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told. - console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`); + if (await announceMerge(client, repo, pull)) pace.miss(`the merge of ${repo.full_name}#${pull.number}`); } + else announced.add(pull.merge_commit_sha); announced.add(pull.merge_commit_sha); changed = true; } @@ -206,6 +159,73 @@ async function pollMerged(client: GiteaClient): Promise { lastLook = began; } +type RepoOf = { owner: string; name: string; full_name: string; clone_url?: string }; +type PullOf = Awaited>[number]; + +/** **A merge, announced once, whichever path saw it first** (novox/hq issue 729): the poll or the forge's event. */ +async function announceMerge(client: GiteaClient, repo: RepoOf, pull: PullOf): Promise { + if (!pull.merged || !pull.merge_commit_sha || announced.has(pull.merge_commit_sha)) return false; + // Recorded before the first wait, so the other path finds it taken while this one is still asking the forge. + announced.add(pull.merge_commit_sha); + try { + // What it changed, asked for only now: a module is rebuilt because a file inside its own + // directory moved, and without this every module built from a repository is rebuilt for a + // change to any of them (novox/hq 04-ISSUES/131). + const changed = await client.listPullFiles(repo.owner, repo.name, pull.number); + // Which of the directories they are in hold a module at the merge commit (novox/hq issue 278): a + // change inside one is that module's, held or not, and only a file in none is shared code. Not + // said when the list is cut or the forge could not be asked; the mesh then keeps its old rule. + let moduleDirs: string[] | null = null; + if (!changed.truncated) { + try { + moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, pull.merge_commit_sha, changed.paths); + } catch (err) { + console.error(`[gitea] ${repo.full_name}#${pull.number}: which directories hold a module could not be read, ` + + `so the mesh reads its files by the old rule — ${err instanceof Error ? err.message : String(err)}`); + } + } + // The head it merged, and what its head was checked as (novox/hq ADR 0239): the delivery it was, made + // from the forge's word when its owner never heard the head. + let headChecks: Record | null = null; + if (pull.head_sha) { + try { + headChecks = await client.commitStatuses(repo.owner, repo.name, pull.head_sha); + } catch (err) { + console.error(`[gitea] ${repo.full_name}#${pull.number}: its head's statuses could not be read — ${err instanceof Error ? err.message : err}`); + } + } + await emit("pull.merged", { + owner: repo.owner, + repo: repo.name, + number: pull.number, + title: pull.title, + body: pull.body, + head_sha: pull.head_sha, + ...(headChecks ? { head_checks: headChecks } : {}), + head: pull.head, + base: pull.base, + merge_commit_sha: pull.merge_commit_sha, + merged_at: pull.merged_at, + clone_url: repo.clone_url, + html_url: pull.html_url, + paths: changed.paths, + paths_truncated: changed.truncated, + // Which of them the merge deleted (novox/hq ADR 0236): a module whose manifest went is forgotten, + // not built. + removed: changed.removed, + ...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}), + }); + // Said, because a trigger that fires silently is indistinguishable from one that did not + // fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told. + console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`); + } catch (err) { + announced.delete(pull.merge_commit_sha); + throw err; + } + keepAnnounced(); + return true; +} + // **Every new head of an open pull request is announced, once** (novox/hq to-be 45 §9): the mesh checks it // against every machine of its facts before it merges. Kept beside the merges' record, so a restart // announces nothing twice; the first look on a machine with no record announces only what moved in the @@ -228,6 +248,31 @@ function keepHeads(): void { writeFileSync(tmp, JSON.stringify({ heads })); renameSync(tmp, pullsRecord); } +/** **A head, announced once, whichever path saw it first** (novox/hq issue 729): the poll or the forge's event. */ +async function announceHead(client: GiteaClient, repo: RepoOf, pull: OpenOf): Promise { + const key = `${repo.full_name}#${pull.number}`; + const sha = String(pull.head_sha ?? ""); + const before = heads[key]; + // Claimed before the first wait, so the other path finds it taken while this one is still announcing. + if (!claimHead(heads, key, sha)) return false; + try { + // The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5). + await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url ?? "", pull)); + } catch (err) { + if (before === undefined) delete heads[key]; + else heads[key] = before; + throw err; + } + keepHeads(); + // Said, so an operator knows the mesh was asked to check it. + console.log(`[gitea] announced ${key} at ${sha.slice(0, 8)} to be checked before it merges`); + // Pending until the verdict comes, so the pull request says a check is running rather than nothing — and + // where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446). + await gates.pending(repo.owner, repo.name, sha); + return true; +} +type OpenOf = Awaited>[number]; + let lastPullLook = ""; // **A sweep now and then looks at every repository** (novox/hq issue 347), whatever moved: the floor is a // heuristic, and a heuristic that misses is caught by the next sweep rather than never. @@ -259,15 +304,8 @@ async function pollPulls(client: GiteaClient): Promise { const key = `${repo.full_name}#${pull.number}`; const fresh = primedPulls || (!!pull.updated_at && pull.updated_at > dayAgo); if (fresh) { - // The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5). - await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url, pull)); - // Said, so an operator knows the mesh was asked to check it. - console.log(`[gitea] announced ${key} at ${String(pull.head_sha).slice(0, 8)} to be checked before it merges`); - // Pending until the verdict comes, so the pull request says a check is running rather than nothing — and - // where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446). - await gates.pending(repo.owner, repo.name, String(pull.head_sha)); - } - heads[key] = String(pull.head_sha); + if (await announceHead(client, repo, pull)) pace.miss(`${key} at ${String(pull.head_sha).slice(0, 8)}`); + } else heads[key] = String(pull.head_sha); changed = true; } if (await trunksMoved(client, repo.owner, repo.name)) changedTrunks = true; @@ -362,22 +400,6 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`); } -// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the -// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection -// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads -// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is -// required would let an untested repository merge. -async function repoCheckFacts(client: GiteaClient, c: Checked, base: string | undefined): Promise { - if (c["repo-check"]?.verdict !== "warning") return {}; - const defined = await client.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined); - if (defined !== false) return { defined }; - const rules = await client.branchProtections(c.owner, c.repo).catch(() => undefined); - if (!rules) return { defined }; - const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === (base || "main")); - const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT); - return { defined, required }; -} - // **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words, // where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last // (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending. @@ -391,10 +413,16 @@ const gates = new GateWriter({ // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own // process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to // reach, nothing is served, as the tools' bundle serves nothing then. -registerModuleTools("gitea", () => (gitea ? [commitStatusTool({ +/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what + * gitea_events says of them (webhook.ts). */ +const pace = new Pace(); +if (!gitea) pace.unheard("the forge is not watched: no way to a token", false); + +// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). +registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({ setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), gates, -}), mergeTool(gitea, { gates })] : [])); +}), mergeTool(gitea, { gates })] : [])]); // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing. @@ -413,12 +441,65 @@ async function buildStarted(event: { body: unknown }): Promise { } } +/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */ +async function actOn(client: GiteaClient, event: ForgeEvent): Promise { + const b = event.body as { owner?: string; repo?: string; number?: number }; + if (!b.owner || !b.repo || !b.number) return; + const head = ["forge.pull-request.opened", "forge.pull-request.synchronized", "forge.pull-request.reopened"].includes(event.name); + const merged = event.name === "forge.pull-request.merged"; + if (!head && !merged) return; + const [repo, pull] = await Promise.all([client.getRepo(b.owner, b.repo), client.getPullRequest(b.owner, b.repo, b.number)]); + if (head && pull.state === "open") await announceHead(client, repo, pull); + if (merged) await announceMerge(client, repo, pull); +} + +/** + * **The receiver, and the forge's one system webhook pointing at it** (issue 729). The secret is minted and kept beside + * the token; the webhook is made or corrected with the admin account (ADR 0266: the tools' token holds no admin scope). + * Answers whether events are heard; a forge that refuses is said, and the polls keep their pace. + */ +async function listenForForge(client: GiteaClient): Promise { + const stateDir = process.env.MESH_GITEA_STATE_DIR; + const url = process.env.MESH_GITEA_HOOK_URL; + if (!stateDir || !url) { + console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge"); + pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false); + return false; + } + try { + const secret = keptSecret(join(stateDir, "webhook.secret")); + const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT); + const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0"; + const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery), + refused: (reason, from) => pace.refused(reason, from), + publish: (event) => emit(event.name, event.body), + act: (event) => actOn(client, event) }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => resolve()); + }); + const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); + pace.listening(host, port, hook.url, hook); + console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` + + `at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` + + `${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` + + `${hook.events.length} event families, signed`); + return true; + } catch (err) { + const why = err instanceof Error ? err.message : String(err); + pace.unheard(why); + console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`); + return false; + } +} + if (gitea) { const client = gitea; // A poll that fails says so once, not once a minute: the same reason repeating (the forge not up // yet, the admin account refused on a restored forge) is one fact, and a recovery is worth a line. let failing: string | null = null; - const tick = (fn: () => Promise, everyMs: number): void => { + const tick = (fn: () => Promise, every: number | (() => number)): void => { + const everyMs = (): number => (typeof every === "number" ? every : every()); // **One pass at a time** (novox/hq issue 250): the next pass is scheduled when this one has ended, so a // pass that outlasts its interval delays the next instead of running beside it — two passes at once // could each announce the same merge before either recorded it. @@ -433,12 +514,15 @@ if (gitea) { if (why !== failing) console.error(`[gitea] not watching until this clears — ${why}`); failing = why; }) - .finally(() => setTimeout(run, everyMs)); + .finally(() => setTimeout(run, everyMs())); run(); }; + // **The forge's events first, the polls as the safety net** (novox/hq issue 729): while verified deliveries arrive the + // polls slow to every five minutes and announce only what no event said (Pace); otherwise they keep 30 s. + await listenForForge(client); tick(() => pollRepos(client), 60_000); - tick(() => pollMerged(client), 30_000); - tick(() => pollPulls(client), 30_000); + tick(() => pollMerged(client), () => pace.every()); + tick(() => pollPulls(client), () => pace.every()); await on("mesh-controller.checked", (event) => { const c = (event.body ?? {}) as Checked; // A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts). diff --git a/modules/gitea/internal/forgehttp/forgehttp.go b/modules/gitea/internal/forgehttp/forgehttp.go index 9b76e553..3b3e8c89 100644 --- a/modules/gitea/internal/forgehttp/forgehttp.go +++ b/modules/gitea/internal/forgehttp/forgehttp.go @@ -1,7 +1,17 @@ // Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge, -// cmd/npm-registry): the forge on its own machine, as its admin account, whose password the vault delivered -// into a file this module owns (novox/hq ADR 0086), by basic authentication. The password is read when a call -// is made and never logged, answered or put in an error. One copy for both bundles (novox/hq issue 539). +// cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539). +// +// Two ways in, by what the manifest hands the bundle: +// +// - **as the tools' account** (novox/hq ADR 0365 §2), when MESH_GITEA_TOKEN_FILE names the token the module +// keeps for it (token.ts mints it, for `mesh-tools`, and keeps it at 0600 with its scopes line beside it). A +// token whose scopes line does not say it is the tools' account's — one kept from before, the administrator's — +// is never sent: the call fails, saying so, until the module has minted the account's own. This bundle mints +// nothing; a token the forge rejects is renewed by the module's next call, and said here. +// - **as the admin account**, whose password the vault delivered into a file this module owns (novox/hq ADR +// 0086), by basic authentication, when no token file is named. +// +// A credential is read when a call is made and never logged, answered or put in an error. package forgehttp import ( @@ -25,12 +35,31 @@ type Client struct { URL string // e.g. http://127.0.0.1:3000 User string PasswordFile string - HTTP *http.Client // when nil, a client with Timeout - Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero + // TokenFile, when set, is the tools' account's token (MESH_GITEA_TOKEN_FILE), sent instead of the admin login; + // Account the account its scopes line must name (MESH_GITEA_TOOLS_USER, default mesh-tools). + TokenFile string + Account string + HTTP *http.Client // when nil, a client with Timeout + Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero } -// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. +// ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts). +const ToolsAccount = "mesh-tools" + +// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle +// handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login. func FromEnv() (Client, []string) { + if file := os.Getenv("MESH_GITEA_TOKEN_FILE"); file != "" { + c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: file, + Account: os.Getenv("MESH_GITEA_TOOLS_USER")} + if c.Account == "" { + c.Account = ToolsAccount + } + if c.URL == "" { + return c, []string{"MESH_GITEA_URL"} + } + return c, nil + } c := Client{ URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), User: os.Getenv("MESH_GITEA_ADMIN_USER"), @@ -80,14 +109,45 @@ func (c *Client) password() (string, error) { return strings.TrimSpace(string(raw)), nil } +// token is the tools' account's token, refused unless its scopes line names that account. +func (c *Client) token() (string, error) { + raw, err := os.ReadFile(c.TokenFile) + if err != nil { + return "", fmt.Errorf("the tools' token is not kept yet (the module mints it for %s at its first call): %w", c.Account, err) + } + line, err := os.ReadFile(c.TokenFile + ".scopes") + if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) { + return "", fmt.Errorf("the token kept at %s is not %s's (novox/hq ADR 0365): the module mints that account's own at its next call; nothing was sent", c.TokenFile, c.Account) + } + token := strings.TrimSpace(string(raw)) + if token == "" { + return "", fmt.Errorf("the token kept at %s is empty; nothing was sent", c.TokenFile) + } + return token, nil +} + +// authorize puts the bundle's credential on a request: the tools' token, or the admin login. +func (c *Client) authorize(req *http.Request) error { + if c.TokenFile != "" { + token, err := c.token() + if err != nil { + return err + } + req.Header.Set("Authorization", "token "+token) + return nil + } + password, err := c.password() + if err != nil { + return err + } + req.SetBasicAuth(c.User, password) + return nil +} + // Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit; // 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it // exactly: a cut body is refused, never read as if it were whole. func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { - password, err := c.password() - if err != nil { - return nil, err - } var body io.Reader if payload != nil { raw, err := json.Marshal(payload) @@ -100,7 +160,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit if err != nil { return nil, err } - req.SetBasicAuth(c.User, password) + if err := c.authorize(req); err != nil { + return nil, err + } req.Header.Set("Accept", "application/json") if payload != nil { req.Header.Set("Content-Type", "application/json") @@ -118,6 +180,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) } switch { + case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "": + return nil, fmt.Errorf("the forge rejected the tools' token for %s (401): the module mints it afresh at its next call; nothing was done: %w", + method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"}) case res.StatusCode == http.StatusNotFound: return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound) case res.StatusCode >= 300: diff --git a/modules/gitea/merge.ts b/modules/gitea/merge.ts index 05ee77a2..9c8e2570 100644 --- a/modules/gitea/merge.ts +++ b/modules/gitea/merge.ts @@ -54,9 +54,10 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition const method = args.method ? String(args.method) : "merge"; if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`); const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch); - // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left - // to the forge, whose administrator — the account this tool's token is minted for — may merge past a - // rule that lets it. A rule that cannot be read is no rule passed. + // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here as well as by the + // forge, which holds the tools' own account to it (novox/hq ADR 0365 §2). The protection is the one the forge + // applies to the base, read from the branch: the rules' own list is a repository administrator's. A rule that + // cannot be read is no rule passed. // **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d, // force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the // head was and nothing told it which head the review had approved. The caller names it; a head that moved @@ -80,7 +81,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` + `${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`); } - const rule = await gitea.branchProtection(owner, repo, pull.base); + const rule = await gitea.appliedProtection(owner, repo, pull.base); // Only into a branch whose protection requires the checks: a pull request whose base was moved to a // branch nothing guards would merge past every check (novox/hq ADR 0266). if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) { diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 069d7163..987cd1d6 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -42,7 +42,47 @@ "issue.opened", "pull.merged", "pull.updated", - "pull.closed" + "pull.closed", + "forge.push", + "forge.create", + "forge.delete", + "forge.fork", + "forge.status", + "forge.pull-request.opened", + "forge.pull-request.reopened", + "forge.pull-request.synchronized", + "forge.pull-request.edited", + "forge.pull-request.closed", + "forge.pull-request.merged", + "forge.pull-request.labeled", + "forge.pull-request.unlabeled", + "forge.pull-request.assigned", + "forge.pull-request.unassigned", + "forge.pull-request.reviewed", + "forge.pull-request.review-requested", + "forge.pull-request.commented", + "forge.pull-request.changed", + "forge.issue-comment.created", + "forge.issue-comment.edited", + "forge.issue-comment.deleted", + "forge.issues.opened", + "forge.issues.reopened", + "forge.issues.closed", + "forge.issues.edited", + "forge.issues.labeled", + "forge.issues.unlabeled", + "forge.issues.assigned", + "forge.issues.unassigned", + "forge.issues.changed", + "forge.release.published", + "forge.release.updated", + "forge.release.deleted", + "forge.release.changed", + "forge.repository.created", + "forge.repository.deleted", + "forge.repository.changed", + "forge.wiki.changed", + "forge.other" ], "consumes": [ "mesh-controller.checked", @@ -61,7 +101,14 @@ "port": 22, "protocol": "tcp", "from": "mesh", - "why": "git over ssh, gitea's own unmodified sshd. Published on the machine's own side at 222, the mesh's fixed public convention \u2014 not 22, which the machine's own daemon holds and a module does not take" + "why": "git over ssh, gitea's own unmodified sshd. Published on the machine's own side at 222, the mesh's fixed public convention — not 22, which the machine's own daemon holds and a module does not take" + }, + { + "name": "forge-events", + "port": 3047, + "protocol": "tcp", + "from": "machine", + "why": "the forge's system webhook, every event signed with a secret the module keeps, from the forge's container on the docker bridge to the module's receiver at the bridge's gateway 172.17.0.1 (novox/hq issue 729)" } ], "serves": { @@ -112,8 +159,8 @@ "settings": { "ready-authors": { "kind": "preference", - "default": "mesh-admin jschoubben", - "why": "the forge accounts whose comment \"ready at \" marks a pull request ready for mesh-delivery to merge, separated by spaces: the mesh's own forge account and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word" + "default": "mesh-tools mesh-admin jschoubben", + "why": "the forge accounts whose comment \"ready at \" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word" } }, "resources": [ @@ -153,7 +200,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\nGITEA__database__DB_TYPE=postgres\nGITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nGITEA__database__NAME=${bound:postgres-database:as}\nGITEA__database__USER=${bound:postgres-database:as}\nGITEA__database__PASSWD=${secret:postgres-database}\n" + "content": "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\nGITEA__database__DB_TYPE=postgres\nGITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nGITEA__database__NAME=${bound:postgres-database:as}\nGITEA__database__USER=${bound:postgres-database:as}\nGITEA__database__PASSWD=${secret:postgres-database}\nGITEA__webhook__ALLOWED_HOST_LIST=172.17.0.1\n" }, { "id": "data", @@ -261,9 +308,13 @@ "MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json", "MESH_GITEA_ADMIN_USER": "mesh-admin", "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", + "MESH_GITEA_TOOLS_USER": "mesh-tools", "MESH_GITEA_STATE_DIR": "${dir:runtime-state}", "MESH_RECEIVES": "${dir:grants}/npm.json", - "MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors" + "MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors", + "MESH_GITEA_HOOK_URL": "http://172.17.0.1:3047", + "MESH_GITEA_HOOK_PORT": "3047", + "MESH_GITEA_HOOK_HOST": "172.17.0.1" } }, { @@ -295,8 +346,8 @@ ], "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", - "MESH_GITEA_ADMIN_USER": "mesh-admin", - "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret" + "MESH_GITEA_TOOLS_USER": "mesh-tools", + "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token" } } ] diff --git a/modules/gitea/pulls.ts b/modules/gitea/pulls.ts index a51156a1..2c3a549d 100644 --- a/modules/gitea/pulls.ts +++ b/modules/gitea/pulls.ts @@ -9,7 +9,7 @@ // // Pure functions here, so they are tested without a forge; index.ts does the asking and the setting. -import type { CommitStatus, GiteaPull, GiteaRepo } from "./client.js"; +import type { CommitStatus, GiteaClient, GiteaPull, GiteaRepo } from "./client.js"; import { movedSince } from "./client.js"; /** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */ @@ -176,6 +176,24 @@ export interface RepoCheckFacts { required?: boolean; } +// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the +// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection +// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads +// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is +// required would let an untested repository merge. The base's protection is the one the forge applies, read +// from the branch: the rules' own list is a repository administrator's, which the tools' account is not +// (novox/hq ADR 0365). +export async function repoCheckFacts( + forge: Pick, c: Checked, base: string | undefined): Promise { + if (c["repo-check"]?.verdict !== "warning") return {}; + const defined = await forge.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined); + if (defined !== false) return { defined }; + const rule = await forge.appliedProtection(c.owner, c.repo, base || "main").catch(() => undefined); + if (rule === undefined) return { defined }; + const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT); + return { defined, required }; +} + /** The description of a required repository check that is not defined. */ export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " + "a person adds one, or lifts the requirement from the branch's protection"; diff --git a/modules/gitea/test/announcing.test.ts b/modules/gitea/test/announcing.test.ts index 01ce7a05..a7a08c58 100644 --- a/modules/gitea/test/announcing.test.ts +++ b/modules/gitea/test/announcing.test.ts @@ -47,11 +47,18 @@ test("every open pull request is read with what its base requires and what its h if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) { return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" }); } - if (url.startsWith("/api/v1/repos/novox/hq/branch_protections")) { - return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/repo-check"] }]); + // The base's protection, read from the branch as the tools' account may (novox/hq ADR 0365); the rules' list is refused it. + if (url.includes("/branch_protections")) { + res.statusCode = 403; + return res.end(JSON.stringify({ message: "the rules are a repository administrator's" })); } - if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branch_protections")) { - return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }]); + if (url.startsWith("/api/v1/repos/novox/hq/branches/main")) { + return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true, + status_check_contexts: ["mesh/repo-check"] }); + } + if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branches/main")) { + return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true, + status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }); } if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] }); if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending", @@ -251,6 +258,8 @@ test("a long thread hides nothing: every page of the timeline is read", async () test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => { const { readyAuthors } = await import("../client.ts"); assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]); - assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }), ["mesh-admin"]); - assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n"), ["mesh-admin"]); + // The tools' account and the admin's, whose marks were made before ADR 0365 (issue 469). + assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }, "mesh-tools"), ["mesh-tools", "mesh-admin"]); + assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n", "mesh-tools"), ["mesh-tools", "mesh-admin"]); + assert.deepEqual(readyAuthors(undefined, "mesh-admin", undefined, "mesh-tools"), ["mesh-tools", "mesh-admin"]); }); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index f14b50ec..567ad023 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -30,6 +30,9 @@ const state = { // What the tools said on the bus. emitted: [] as { type: string; body: any }[], rules: [] as any[], + // Whether the token is a repository administrator's. The tools' account is not (novox/hq ADR 0365): the forge + // refuses it the rules' list, as gitea's reqAdmin does, and it reads a branch's protection from the branch. + admin: false, lastCommit: "2026-01-01T00:00:00Z", // The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's // record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the @@ -59,6 +62,7 @@ beforeEach(() => { state.trunk = TRUNK; state.emitted = []; state.rules = [protectedMain()]; + state.admin = false; state.head = REVIEWED; state.landed = ""; state.merged = false; @@ -87,8 +91,18 @@ const server = createServer(async (req, res) => { if (method !== "GET") state.sent.push({ method, path, body }); const repo = "/repos/novox/mesh-catalog"; if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" }); + if (path.startsWith(`${repo}/branch_protections`) && !state.admin) { + return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + } if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules); - if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } }); + const branch = method === "GET" && path.startsWith(`${repo}/branches/`) ? decodeURIComponent(path.slice(`${repo}/branches/`.length)) : null; + if (branch !== null) { + // As the forge says it to any reader of the code: the rule that covers the branch, as the forge matches it. + const rule = (state.rules as any[]).find((r) => ruleCovers(String(r.rule_name ?? r.branch_name), branch)); + return json(res, 200, { name: branch, commit: { id: branch === "main" ? state.trunk : "c0ffee" }, protected: !!rule, + ...(rule ? { effective_branch_protection_name: rule.rule_name ?? rule.branch_name, enable_status_check: !!rule.enable_status_check, + status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: false } : {}) }); + } if (method === "GET" && path === `${repo}/pulls/7/files`) { return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []); } @@ -199,6 +213,7 @@ test("a protection rule covers a branch as the forge decides it: classes, altern }); test("gitea_api still writes what an author writes", async () => { + state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" }); @@ -206,6 +221,7 @@ test("gitea_api still writes what an author writes", async () => { }); test("gitea_branch_protection_set only strengthens", async () => { + state.admin = true; // a repository administrator's token: the tools' account is refused the rules by the forge const set = (args: Record) => tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args }); await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/); @@ -396,6 +412,7 @@ test("gitea_merge_pull_request reads back a fast-forward, and says it cannot rea }); test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => { + state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm }); await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/); await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/); @@ -427,6 +444,7 @@ test("gitea_merge_pull_request merges only into a branch whose protection requir }); test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => { + state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true }); const set = (args: Record) => diff --git a/modules/gitea/test/protection-read.test.ts b/modules/gitea/test/protection-read.test.ts new file mode 100644 index 00000000..d2928ab9 --- /dev/null +++ b/modules/gitea/test/protection-read.test.ts @@ -0,0 +1,59 @@ +// The controller's build-source check asks gitea_branch_protection_get for a repository's rules, which the forge +// answers to a repository's administrator alone. The tools' account is not one (novox/hq ADR 0365 §2), so the read +// goes with the forge's administrator account, and only the read (ADR 0375, issue 772): a protection edit through the +// tools still gets the forge's refusal. + +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { createServer } from "node:http"; + +import { GiteaAdmin, GiteaClient } from "../dist/client.js"; +import { ConfiguredToken } from "../dist/token.js"; +import { getGiteaTools } from "../dist/tools/index.js"; + +const adminBasic = "Basic " + Buffer.from("mesh-admin:secret").toString("base64"); +const rules = [ + { rule_name: "main", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true }, + { branch_name: "release", enable_push: false }, +]; +const seen: string[] = []; + +const server = createServer((req, res) => { + const path = (req.url ?? "").replace(/^\/api\/v1/, ""); + const as = req.headers.authorization === adminBasic ? "admin" : "tools"; + seen.push(`${req.method} ${path} as ${as}`); + const send = (status: number, body: unknown) => { res.writeHead(status, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); }; + if (path.includes("/branch_protections")) { + if (as !== "admin") return send(403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + if (req.method === "GET") return send(200, rules); + } + send(404, { message: "not found" }); +}); +await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); +after(() => new Promise((r) => server.close(() => r()))); +const { port } = server.address() as { port: number }; +const url = `http://127.0.0.1:${port}`; +const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret")); +const tool = (name: string) => tools.find((t) => t.name === name)!; + +test("gitea_branch_protection_get answers every rule, read with the administrator account the tools' token cannot read with", async () => { + const out: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller" }); + assert.deepEqual(out.rules.map((r: any) => r.rule), ["main", "release"]); +}); + +test("gitea_branch_protection_get answers the rule for a branch, null when it has none", async () => { + const main: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "main" }); + assert.equal(main.branch, "main"); + assert.equal(main.rule.rule, "main"); + assert.equal(main.rule.admin_may_override, false); + const none: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "dev" }); + assert.equal(none.rule, null); +}); + +test("gitea_branch_protection_set still gets the forge's refusal, as the tools' account", async () => { + await assert.rejects( + tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main", status_checks: "mesh/merge-gate" }), + /403/, + ); + assert.ok(!seen.some((s) => /^(PATCH|POST|PUT|DELETE) .*branch_protections.* as admin$/.test(s)), seen.join("\n")); +}); diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 750c8122..d72202f2 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -10,7 +10,7 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -29,6 +29,12 @@ interface Forge { mints: number; lastScopes: string[] | null; tokens: Map; + /** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */ + twins: { id: number; name: string; sha1: string }[]; + /** How each DELETE named its token: "id" or "name". */ + deletedBy: string[]; + /** Plant a token by name, beside any other of the same name, as a raced mint would. */ + plant(name: string): void; scopesOf: Map; admins: Map; pullState: string; @@ -39,9 +45,14 @@ interface Forge { function fakeForge(): Promise { const forge = { + mintedFor: "", mints: 0, lastScopes: null as string[] | null, tokens: new Map(), // name -> value + twins: [] as { id: number; name: string; sha1: string }[], + deletedBy: [] as string[], + ids: new Map(), // value -> id, as gitea numbers its tokens + nextId: 0, scopesOf: new Map(), // value -> scopes, so a route can enforce them like gitea does admins: new Map([[ADMIN, PASSWORD]]), pullState: "open", @@ -73,7 +84,14 @@ function fakeForge(): Promise { const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/); if (tokens) { const user = basic(req); - if (user === null || user !== decodeURIComponent(tokens[1])) return json(res, 401, { message: "auth required" }); + // gitea's reqSelfOrAdmin: the account itself, or the administrator for another (the tools' account, ADR 0365). + if (user === null || (user !== decodeURIComponent(tokens[1]) && user !== ADMIN)) return json(res, 401, { message: "auth required" }); + // This fake keeps one account's tokens: the tools' account's when the module mints for it (ADR 0365), and the + // admin's own list is then empty — nothing of it is retired. + if (decodeURIComponent(tokens[1]) === ADMIN && forge.mintedFor === "mesh-tools") { + return req.method === "GET" ? json(res, 200, []) : json(res, 404, { message: "token not found" }); + } + if (req.method === "POST") forge.mintedFor = decodeURIComponent(tokens[1]); if (req.method === "POST") { const { name, scopes } = await body(req); if (forge.tokens.has(name)) return json(res, 400, { message: "token name has already been used" }); @@ -82,12 +100,35 @@ function fakeForge(): Promise { const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`; forge.tokens.set(name, sha1); forge.scopesOf.set(sha1, scopes); - return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) }); + forge.ids.set(sha1, ++forge.nextId); + return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) }); + } + // gitea's list: every token of the account, a page at a time (at most three to a page here, so a + // client that reads only the first page misses some), the values never shown. + const listed = (): { id: number; name: string; sha1: string }[] => [ + ...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })), + ...forge.twins, + ]; + if (req.method === "GET" && !tokens[2]) { + const page = Math.max(1, Number(url.searchParams.get("page") ?? "1")); + const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3"))); + const all = listed(); + res.setHeader("X-Total-Count", String(all.length)); + return json(res, 200, all.slice((page - 1) * limit, page * limit) + .map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] }))); } if (req.method === "DELETE" && tokens[2]) { - const name = decodeURIComponent(tokens[2]); - if (!forge.tokens.has(name)) return json(res, 404, { message: "token not found" }); - forge.tokens.delete(name); + // gitea: "identified by ID and if not available by name" — and a name held twice is refused. + const key = decodeURIComponent(tokens[2]); + const byId = /^\d+$/.test(key); + const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key)); + if (matches.length === 0) return json(res, 404, { message: "token not found" }); + if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` }); + const t = matches[0]; + forge.deletedBy.push(byId ? "id" : "name"); + const twin = forge.twins.findIndex((w) => w.id === t.id); + if (twin >= 0) forge.twins.splice(twin, 1); + else forge.tokens.delete(t.name); return json(res, 204, null); } return json(res, 405, { message: "method not allowed" }); @@ -153,6 +194,17 @@ function fakeForge(): Promise { { full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" }, ]); } + // What setting up the tools' account asks with the admin login (tools-account.ts, ADR 0365): answered plainly. + if (basic(req) === ADMIN) { + const p = url.pathname; + if (req.method === "GET" && p === "/api/v1/user") return json(res, 200, { login: ADMIN, is_admin: true }); + if (req.method === "GET" && p === "/api/v1/users/mesh-tools") return json(res, 200, { login: "mesh-tools", is_admin: false }); + if (req.method === "POST" && p === "/api/v1/admin/users") return json(res, 422, { message: "user already exists" }); + if (req.method === "PATCH" && p === "/api/v1/admin/users/mesh-tools") return json(res, 200, { login: "mesh-tools" }); + if (req.method === "GET" && p === "/api/v1/orgs/novox/teams") return json(res, 200, [{ id: 7, name: "mesh-tools" }]); + if (req.method === "PATCH" && p === "/api/v1/teams/7") return json(res, 200, { id: 7 }); + if (req.method === "PUT" && p === "/api/v1/teams/7/members/mesh-tools") return json(res, 204, null); + } const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/); if (adminUser && req.method === "PATCH") { const h = req.headers.authorization ?? ""; @@ -181,6 +233,14 @@ function fakeForge(): Promise { get pullTitle() { return forge.pullTitle; }, get branchDeleted() { return forge.branchDeleted; }, tokens: forge.tokens, + twins: forge.twins, + deletedBy: forge.deletedBy, + plant: (name: string) => { + const sha1 = `planted-${++forge.nextId}`; + forge.ids.set(sha1, forge.nextId); + if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 }); + else forge.tokens.set(name, sha1); + }, scopesOf: forge.scopesOf, admins: forge.admins, close: () => new Promise((r) => server.close(() => r())), @@ -335,6 +395,56 @@ test("concurrent first calls share one mint", async () => { assert.equal(forge.mints, before + 1); }); +test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => { + const { env, file, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept + forge.plant("mesh-tools"); + forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on + forge.deletedBy.length = 0; + const before = forge.mints; + + const repos = await minted(env, logs).listRepos(); + + assert.equal(repos.length, 1); + assert.equal(forge.mints, before + 1); + assert.deepEqual(forge.deletedBy, ["id", "id"]); + assert.equal(forge.twins.length, 0); + assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]); + assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n"); + assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n")); +}); + +test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => { + const { env, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + const before = forge.mints; + // Two sources over one state directory, as the tools bundle and the watcher are two processes. + const first = minted(env, logs); + const second = minted(env, logs); + + await Promise.all([first.listRepos(), second.listRepos()]); + + assert.equal(forge.mints, before + 1); + assert.equal(forge.twins.length, 0); + assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n")); +}); + +test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => { + const { env, file, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true }); + await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds + const before = forge.mints; + + assert.equal((await minted(env, logs).listRepos()).length, 1); + assert.equal(forge.mints, before + 1); + await assert.rejects(stat(`${file}.lock`), /ENOENT/); +}); + test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => { const { env, file, logs } = await delivered(forge); forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts new file mode 100644 index 00000000..efc6dcfa --- /dev/null +++ b/modules/gitea/test/tools-account.test.ts @@ -0,0 +1,332 @@ +// The forge's tools act as an account of their own, not the site administrator (novox/hq ADR 0365 §2, issue 469): +// the token the tools and the watcher use is minted for `mesh-tools`, a plain account made by the module with the +// administrator's login and given write on the mesh's repositories through one team; the administrator's own +// token by that name goes; a token kept from before, the administrator's, is minted again for the account. +// +// The forge is a fake that keeps accounts, their tokens and the organisation's teams apart, and answers as gitea +// does: an administrator may mint for another account (reqSelfOrAdmin), anyone else only for itself; a protection +// edit needs the repository's administration, which neither a plain account nor its write team holds. + +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { MintedToken } from "../dist/token.js"; +import { GiteaClient } from "../dist/client.js"; +import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js"; +import { repoCheckFacts } from "../dist/pulls.js"; + +const ADMIN = "mesh-admin"; +const PASSWORD = "the-vault-minted-this"; + +interface Account { admin: boolean; password: string; active: boolean } +interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] } +interface Team { id: number; name: string; units: any; members: Set } + +function fakeForge() { + const users = new Map([[ADMIN, { admin: true, password: PASSWORD, active: true }]]); + const tokens: Token[] = []; + const teams: Team[] = []; + const calls: string[] = []; + // What the forge does wrong when a test says so: a token mint or delete it fails, a team another process made + // between this one's look and its create (gitea answers the create 422). + const faults = { mint: false, deleteToken: false, teamRace: false }; + let next = 0; + const json = (res: ServerResponse, status: number, body: unknown): void => { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(body === null ? "" : JSON.stringify(body)); + }; + const read = (req: IncomingMessage): Promise => + new Promise((resolve) => { + let t = ""; + req.on("data", (c) => (t += c)); + req.on("end", () => resolve(t ? JSON.parse(t) : null)); + }); + const who = (req: IncomingMessage): string | null => { + const h = req.headers.authorization ?? ""; + if (h.startsWith("Basic ")) { + const [u, p] = Buffer.from(h.slice(6), "base64").toString().split(":"); + return users.get(u)?.password === p ? u : null; + } + if (h.startsWith("token ")) return tokens.find((t) => t.sha1 === h.slice(6))?.user ?? null; + return null; + }; + const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true; + + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? "/", "http://fake"); + const p = url.pathname.replace(/^\/api\/v1/, ""); + const me = who(req); + calls.push(`${req.method} ${p} as ${me}`); + if (me === null) return json(res, 401, { message: "unauthorized" }); + + let m = p.match(/^\/users\/([^/]+)\/tokens(?:\/(\d+))?$/); + if (m) { + const owner = decodeURIComponent(m[1]); + if (!(req.headers.authorization ?? "").startsWith("Basic ")) return json(res, 401, { message: "basic auth required" }); + if (owner !== me && !isAdmin(me)) return json(res, 403, { message: "reqSelfOrAdmin" }); + if (!users.has(owner)) return json(res, 404, { message: "user does not exist" }); + if (req.method === "POST") { + if (faults.mint) return json(res, 500, { message: "the forge is not well" }); + const { name, scopes } = await read(req); + if (tokens.some((t) => t.user === owner && t.name === name)) return json(res, 400, { message: "token name has already been used" }); + const t = { id: ++next, name, sha1: `sha-${owner}-${next}`, user: owner, scopes }; + tokens.push(t); + return json(res, 201, { id: t.id, name, sha1: t.sha1, scopes }); + } + if (req.method === "GET") { + const page = Number(url.searchParams.get("page") ?? "1"); + const mine = tokens.filter((t) => t.user === owner); + return json(res, 200, page === 1 ? mine.map((t) => ({ id: t.id, name: t.name, sha1: "" })) : []); + } + if (req.method === "DELETE" && m[2]) { + if (faults.deleteToken && owner === ADMIN) return json(res, 500, { message: "the forge is not well" }); + const i = tokens.findIndex((t) => t.user === owner && t.id === Number(m![2])); + if (i < 0) return json(res, 404, { message: "not found" }); + tokens.splice(i, 1); + return json(res, 204, null); + } + } + if (req.method === "GET" && p === "/user") return json(res, 200, { login: me, is_admin: isAdmin(me) }); + m = p.match(/^\/users\/([^/]+)$/); + if (m && req.method === "GET") { + const u = users.get(decodeURIComponent(m[1])); + if (!u) return json(res, 404, { message: "user does not exist" }); + return json(res, 200, { login: m[1], is_admin: u.admin, active: u.active, prohibit_login: false }); + } + if (p.startsWith("/admin/") && !isAdmin(me)) return json(res, 403, { message: "must be a site administrator" }); + if (req.method === "POST" && p === "/admin/users") { + const b = await read(req); + if (users.has(b.username)) return json(res, 422, { message: "user already exists" }); + if (!b.password || b.password.length < 8) return json(res, 422, { message: "password is too short" }); + users.set(b.username, { admin: b.admin === true, password: b.password, active: true }); + return json(res, 201, { login: b.username, is_admin: b.admin === true }); + } + m = p.match(/^\/admin\/users\/([^/]+)$/); + if (m && req.method === "PATCH") { + const u = users.get(decodeURIComponent(m[1])); + if (!u) return json(res, 404, { message: "user does not exist" }); + const b = await read(req); + if (b.admin !== undefined) u.admin = b.admin; + if (b.active !== undefined) u.active = b.active; + return json(res, 200, { login: m[1], is_admin: u.admin }); + } + if (p === "/orgs/novox/teams") { + if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" }); + if (req.method === "GET") return json(res, 200, teams.map((t) => ({ id: t.id, name: t.name }))); + const b = await read(req); + const t = { id: ++next, name: b.name, units: b, members: new Set() }; + teams.push(t); + if (faults.teamRace) return json(res, 422, { message: "team already exists" }); + return json(res, 201, { id: t.id, name: t.name }); + } + m = p.match(/^\/teams\/(\d+)(?:\/members\/([^/]+))?$/); + if (m) { + if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" }); + const t = teams.find((x) => x.id === Number(m![1])); + if (!t) return json(res, 404, { message: "team does not exist" }); + if (req.method === "PATCH") { t.units = await read(req); return json(res, 200, { id: t.id }); } + if (req.method === "PUT" && m[2]) { t.members.add(decodeURIComponent(m[2])); return json(res, 204, null); } + } + // Repository administration: a site administrator, or a team member whose team administers the repository. + m = p.match(/^\/repos\/novox\/([^/]+)\/branches\/(.+)$/); + if (m && req.method === "GET") { + if (m[2] === "main") return json(res, 200, { name: "main", protected: true, effective_branch_protection_name: "main", + enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], user_can_push: false }); + if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false }); + return json(res, 404, { message: "branch does not exist" }); + } + // gitea's reqAdmin on the whole group: reading the rules is administration too. + m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/.*)?$/); + if (m) { + const admins = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.permission === "admin"); + if (!admins) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + return json(res, 201, { rule_name: "main" }); + } + m = p.match(/^\/repos\/novox\/([^/]+)\/statuses\/([0-9a-f]+)$/); + if (m && req.method === "POST") { + const writes = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.units_map?.["repo.code"] === "write"); + if (!writes) return json(res, 403, { message: "write on code required" }); + return json(res, 201, { context: (await read(req)).context, creator: { login: me } }); + } + if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] }); + return json(res, 404, { message: "no such route in the fake" }); + }); + return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[]; + faults: typeof faults; close(): Promise }>((resolve) => { + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as { port: number }; + resolve({ url: `http://127.0.0.1:${port}`, users, tokens, teams, calls, faults, + close: () => new Promise((r) => server.close(() => r())) }); + }); + }); +} + +const forge = await fakeForge(); +after(() => forge.close()); + +async function environment(): Promise { + const dir = await mkdtemp(join(tmpdir(), "gitea-tools-account-")); + const passwordFile = join(dir, "admin.secret"); + await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 }); + return { + MESH_GITEA_URL: forge.url, + MESH_GITEA_ADMIN_USER: ADMIN, + MESH_GITEA_ADMIN_PASSWORD_FILE: passwordFile, + MESH_GITEA_STATE_DIR: join(dir, "state"), + }; +} + +/** A source as a fresh process builds it from the manifest's environment (fromEnv shares one per file). */ +function fresh(env: NodeJS.ProcessEnv): GiteaClient { + return GiteaClient.fromEnv(env); +} + +test("the tools' token is minted for an account of their own, which is no administrator and writes through its team", async () => { + const env = await environment(); + await fresh(env).listRepos(); + + const account = forge.users.get(TOOLS_ACCOUNT); + assert.ok(account, "the module made the tools' account"); + assert.equal(account!.admin, false, "the tools' account is no administrator"); + const held = forge.tokens.filter((t) => t.name === "mesh-tools"); + assert.deepEqual(held.map((t) => t.user), [TOOLS_ACCOUNT], "the one token is the tools' account's, none is the administrator's"); + assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8"), held[0].sha1 + "\n"); + + const team = forge.teams.find((t) => t.name === "mesh-tools"); + assert.ok(team, "the team is made in the mesh's organisation"); + assert.ok(team!.members.has(TOOLS_ACCOUNT)); + assert.equal(team!.units.permission, "write"); + assert.equal(team!.units.includes_all_repositories, true); + assert.equal(team!.units.can_create_org_repo, false, "a repository's maker would administer it"); + assert.deepEqual(team!.units.units_map, TOOLS_TEAM_UNITS.units_map); + assert.ok(!Object.values(team!.units.units_map).includes("admin")); +}); + +test("with the tools' token, the forge writes a status and refuses a protection edit — the forge itself, not a guard", async () => { + const env = await environment(); + const client = fresh(env); + await client.setCommitStatus("novox", "hq", "abc123def4567890abc123def4567890abc12345", { state: "success", context: "mesh/merge-gate", description: "ok" } as any); + await assert.rejects( + client.setBranchProtection("novox", "hq", "main", { statusChecks: ["mesh/merge-gate"] }), + /403/, + ); +}); + +test("what a base's protection asks is read from the branch, which the tools' account may read, not from the rules' list it may not", async () => { + const env = await environment(); + const client = fresh(env); + await assert.rejects(client.branchProtections("novox", "hq"), /403/); + const rule = await client.appliedProtection("novox", "hq", "main"); + assert.deepEqual(rule?.status_check_contexts, ["mesh/merge-gate", "mesh/repo-check"]); + assert.equal(rule?.enable_status_check, true); + assert.equal(rule?.rule_name, "main"); + assert.equal(await client.appliedProtection("novox", "hq", "feat/unguarded"), null); + assert.equal(await client.appliedProtection("novox", "hq", "no/such-branch"), null); +}); + +test("the administrator's token from before goes: a kept admin token is minted again for the tools' account, and the forge drops the old one", async () => { + const env = await environment(); + // What the module kept before ADR 0365: a token of the administrator's own, by the same name, its scopes beside it. + const old = { id: 900, name: "mesh-tools", sha1: "sha-admin-old", user: ADMIN, scopes: ["write:repository", "write:issue", "read:user"] }; + forge.tokens.push(old); + await mkdir(env.MESH_GITEA_STATE_DIR, { recursive: true }); + await writeFile(join(env.MESH_GITEA_STATE_DIR, "token"), old.sha1 + "\n", { mode: 0o600 }); + await writeFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "read:user,write:issue,write:repository\n", { mode: 0o600 }); + + await fresh(env).listRepos(); + + assert.ok(!forge.tokens.some((t) => t.user === ADMIN), "the administrator holds no tools' token any more"); + const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + assert.notEqual(kept, old.sha1); + assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT); + assert.match(await readFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "utf8"), / for mesh-tools\n$/); +}); + +test("a tools' account someone made an administrator is made a plain account again at the next mint", async () => { + const env = await environment(); + forge.users.get(TOOLS_ACCOUNT)!.admin = true; + await fresh(env).listRepos(); + assert.equal(forge.users.get(TOOLS_ACCOUNT)!.admin, false); +}); + +test("the tools never act as the administrator: MESH_GITEA_TOOLS_USER naming it is refused", async () => { + const env = await environment(); + assert.throws(() => MintedToken.fromEnv(forge.url, { ...env, MESH_GITEA_TOOLS_USER: ADMIN }), /names the admin account/); +}); + +test("the manifest hands the tools their account, and the merge-ready authors name it rather than the administrator", async () => { + const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); + const code = manifest.build.artifacts.find((a: any) => a.name === "code"); + assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT); + const authors = String(manifest.settings["ready-authors"].default).split(/\s+/); + assert.ok(authors.includes(TOOLS_ACCOUNT)); + // Kept for now (review of issue 469): a pull request the admin account marked ready before the tools acted as + // their own keeps counting as ready after the delivery. + assert.ok(authors.includes(ADMIN), "marks made by the admin account before ADR 0365 keep counting"); + // The Go forge bundle reads the token the tools' account holds, never the administrator's password. + const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge"); + assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE); + assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined); +}); + +// Review of issue 469: what reads the rules' list is moved to the branch, and nothing leaves the tools without a token. + +test("a repository check said as a warning on a head without merge-check.sh reads the base's protection from the branch", async () => { + const client = fresh(await environment()); + const forgeOf = { holdsFile: async () => false, appliedProtection: client.appliedProtection.bind(client) }; + const c = { owner: "novox", repo: "hq", commit: "abc123def4567890abc123def4567890abc12345", "repo-check": { verdict: "warning" } } as any; + // main requires mesh/repo-check: required, so a head that defines no check fails it for a person to see. + assert.deepEqual(await repoCheckFacts(forgeOf, c, "main"), { defined: false, required: true }); + // A base nothing guards: not required, so the head's repository check passes rather than fails. + assert.deepEqual(await repoCheckFacts(forgeOf, c, "feat/unguarded"), { defined: false, required: false }); +}); + +test("a mint that fails leaves the administrator's old token in place: the tools are never left without one", async () => { + const env = await environment(); + const old = { id: 901, name: "mesh-tools", sha1: "sha-admin-kept", user: ADMIN, scopes: ["read:user"] }; + forge.tokens.push(old); + forge.faults.mint = true; + try { + await assert.rejects(fresh(env).listRepos(), /500/); + } finally { + forge.faults.mint = false; + } + assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's token was deleted before a new one was kept"); + forge.tokens.splice(forge.tokens.indexOf(old), 1); +}); + +test("an old token that cannot be deleted after the mint is said, the new token kept, and deleted at the next start", async () => { + const env = await environment(); + const old = { id: 902, name: "mesh-tools", sha1: "sha-admin-stuck", user: ADMIN, scopes: ["read:user"] }; + forge.tokens.push(old); + forge.faults.deleteToken = true; + try { + await fresh(env).listRepos(); + } finally { + forge.faults.deleteToken = false; + } + const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT, "the account's token is kept and works"); + assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1)); + // The next process reuses the kept token and retires the old one it could not before. + const next = new MintedToken({ url: forge.url, admin: ADMIN, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!, + file: join(env.MESH_GITEA_STATE_DIR, "token"), account: TOOLS_ACCOUNT, log: () => {} }); + assert.equal(await next.current(), kept); + assert.ok(!forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's old token was deleted at the next start"); +}); + +test("a team another process made between the look and the create is found, not a failure", async () => { + forge.teams.splice(0); + forge.faults.teamRace = true; + try { + await fresh(await environment()).listRepos(); + } finally { + forge.faults.teamRace = false; + } + const team = forge.teams.find((t) => t.name === "mesh-tools"); + assert.ok(team?.members.has(TOOLS_ACCOUNT)); +}); diff --git a/modules/gitea/test/webhook.test.ts b/modules/gitea/test/webhook.test.ts new file mode 100644 index 00000000..1160de34 --- /dev/null +++ b/modules/gitea/test/webhook.test.ts @@ -0,0 +1,335 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createHmac } from "node:crypto"; +import type { AddressInfo } from "node:net"; + +// **The forge says every event on the bus** (novox/hq issue 729): a signed delivery is published once and acted on, +// an unsigned or unverified one is refused, and an event and the poll seeing the same head announce it once. + +const secret = "s".repeat(64); +const sign = (body: string, key = secret) => createHmac("sha256", key).update(body).digest("hex"); +const opened = JSON.stringify({ action: "opened", number: 7, pull_request: { number: 7, title: "t", user: { login: "jo" }, + head: { ref: "feat/x", sha: "a".repeat(40) }, base: { ref: "main" }, merged: false }, + repository: { name: "hq", full_name: "novox/hq", owner: { login: "novox" } }, sender: { login: "jo" } }); + +async function serve(published: { name: string }[], acted: { name: string }[]) { + const { receiver, HOOK_PATH } = await import("../webhook.ts"); + const server = receiver({ secret, port: 0, log: () => {}, + publish: async (e) => { published.push(e); }, act: async (e) => { acted.push(e); } }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`; + return { server, url }; +} + +test("a signed event is published on the bus and acted on once", async () => { + const published: { name: string }[] = [], acted: { name: string }[] = []; + const { server, url } = await serve(published, acted); + try { + const headers = { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-1" }; + const res = await fetch(url, { method: "POST", body: opened, headers }); + assert.equal(res.status, 200); + // The same delivery sent again is answered and published nowhere. + const again = await fetch(url, { method: "POST", body: opened, headers }); + assert.equal(again.status, 200); + await server.idle(); + assert.deepEqual(published.map((e) => e.name), ["forge.pull-request.opened"]); + assert.deepEqual(acted.map((e) => e.name), ["forge.pull-request.opened"]); + } finally { server.close(); } +}); + +test("an unsigned event, or one whose signature does not verify, is refused and published nowhere", async () => { + const published: { name: string }[] = [], acted: { name: string }[] = []; + const { server, url } = await serve(published, acted); + try { + const unsigned = await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request" } }); + assert.equal(unsigned.status, 401); + assert.match(await unsigned.text(), /unsigned/); + const forged = await fetch(url, { method: "POST", body: opened, + headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "another secret") } }); + assert.equal(forged.status, 401); + assert.match(await forged.text(), /does not verify/); + const big = "x".repeat(5 * 1024 * 1024 + 1); + const large = await fetch(url, { method: "POST", body: big, + headers: { "X-Gitea-Event": "push", "X-Gitea-Signature": sign(big) } }); + assert.equal(large.status, 413); + await server.idle(); + assert.deepEqual(published, []); + assert.deepEqual(acted, []); + } finally { server.close(); } +}); + +test("an event and the poll seeing the same head announce it once", async () => { + const { claimHead } = await import("../webhook.ts"); + const heads: Record = {}; + const sha = "a".repeat(40); + assert.equal(claimHead(heads, "novox/hq#7", sha), true, "the event claims the head and announces it"); + assert.equal(claimHead(heads, "novox/hq#7", sha), false, "the poll finds it taken"); + assert.equal(claimHead(heads, "novox/hq#7", "b".repeat(40)), true, "a new head is announced again"); +}); + +test("each forge event is named in the module's stream and trimmed", async () => { + const { eventOf } = await import("../webhook.ts"); + const merged = eventOf("pull_request", { action: "closed", pull_request: { number: 3, merged: true, merge_commit_sha: "c".repeat(40), + head: { sha: "a".repeat(40) }, base: { ref: "main" } }, repository: { name: "hq", owner: { login: "novox" } } }); + assert.equal(merged.name, "forge.pull-request.merged"); + assert.equal(merged.body.merge_commit_sha, "c".repeat(40)); + assert.equal(eventOf("pull_request", { action: "label_updated", pull_request: { number: 3 }, label: { name: "mesh/ready" } }).name, + "forge.pull-request.labeled"); + const comment = eventOf("issue_comment", { action: "created", is_pull: true, issue: { number: 3 }, + comment: { id: 9, body: "ready at abc", user: { login: "jo" } } }); + assert.equal(comment.name, "forge.pull-request.commented"); + assert.equal(comment.body.comment, "ready at abc"); + assert.equal(eventOf("push", { ref: "refs/heads/main", after: "d".repeat(40) }).name, "forge.push"); + assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other"); +}); + +// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook +// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default). +// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time. +function fakeForge(start: { id: number; url: string; system: boolean }[] = [], + repoHooks: Record = {}, repoCount = 0) { + const hooks = start.map((h) => ({ ...h, events: ["push"], active: true })); + const repos = [...Object.keys(repoHooks)]; + for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`); + let next = 100; + const calls: string[] = []; + const made: any[] = []; + const page = (items: any[], q: URLSearchParams) => { + const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50); + return items.slice((p - 1) * l, p * l); + }; + const admin = { call: async (method: string, path: string, body?: any) => { + calls.push(`${method} ${path}`); + const u = new URL(path, "http://f"); + const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/); + if (method === "GET" && u.pathname === "/repos/search") { + return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } }; + } + if (repoHook) { + const list = repoHooks[repoHook[1]] ?? []; + if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) }; + const i = list.findIndex((h) => h.id === Number(repoHook[2])); + if (i < 0) return { status: 404, body: {} }; + list.splice(i, 1); + return { status: 204, body: null }; + } + if (method === "GET") { + const type = u.searchParams.get("type") ?? "system"; + const listed = hooks.filter((h) => type === "all" || h.system === (type === "system")); + return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events, + config: { url: h.url, content_type: "json" } })) }; + } + if (method === "POST") { + made.push(body); + hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events, + active: body.active }); + return { status: 201, body: {} }; + } + if (method === "DELETE") { + const id = Number(path.split("/").pop()); + const i = hooks.findIndex((h) => h.id === id); + if (i < 0) return { status: 404, body: {} }; + hooks.splice(i, 1); + return { status: 204, body: null }; + } + return { status: 405, body: {} }; + } }; + return { admin, hooks, calls, made, repoHooks }; +} + +test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const forge = fakeForge(); + const url = "http://h:3047/forge-event"; + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.equal(forge.made.length, 1); + assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository"); + assert.equal(forge.made[0].config.secret, secret); + assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push")); + assert.deepEqual(forge.hooks.map((h) => h.system), [true]); + assert.equal(hook.kind, "system"); + assert.equal(hook.did, "created"); + assert.equal(hook.url, url); + assert.ok(hook.events.includes("pull_request")); +}); + +test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const url = "http://h:3047/forge-event"; + // What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start. + const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]); + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] }); + assert.equal(hook.did, "replaced"); + assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept"); + // A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never + // changes a secret). + const again = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] }); + assert.equal(forge.hooks.filter((h) => h.url === url).length, 1); + assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5")); + // The new one is made before any old one is deleted, so no event falls in a gap. + const post = forge.calls.indexOf("POST /admin/hooks"); + assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post); +}); + +test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const url = "http://h:3047/forge-event"; + // A repository made after the default webhook got its copy, which would send its events a second time; 120 + // repositories, so the copies sit beyond the first page. + const repoHooks: Record = {}; + for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = []; + repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }]; + repoHooks["novox/later"] = [{ id: 33, url }]; + const forge = fakeForge([{ id: 4, url, system: false }], repoHooks); + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] }); + assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept"); + assert.deepEqual(forge.repoHooks["novox/later"], []); + assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read"); +}); + +test("a forge that makes no system webhook is said, never taken as heard", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const forge = fakeForge(); + const admin = { call: async (method: string, path: string, body?: any) => { + // A forge that ignores the flag: what 23:05 met, had the module read back. + if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } }; + return forge.admin.call(method, path, body); + } }; + await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/); +}); + +test("the receiver tells what it verified and what it refused", async () => { + const { receiver, HOOK_PATH } = await import("../webhook.ts"); + const heard: string[] = [], refused: string[] = []; + const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {}, + heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`; + try { + await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } }); + await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } }); + await server.idle(); + assert.deepEqual(heard, ["pull_request d-9"]); + assert.deepEqual(refused, ["its signature does not verify"]); + } finally { server.close(); } +}); + +test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true, + did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } }; + pace.listening("172.17.0.1", 3047, hook.url, hook); + now += 5 * 60_000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.length, 0, "five minutes after the start is not yet silence"); + assert.equal((pace.report() as any).healthy, true); + pace.refused("it is unsigned", "172.17.0.3"); + assert.equal((pace.report() as any).healthy, false, "refused, and none verified"); + now += 6 * 60_000; + pace.miss("the merge of novox/hq#7"); + pace.miss("novox/hq#8 at bbbbbbbb"); + assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once"); + let r = pace.report() as any; + assert.equal(r.healthy, false); + assert.equal(r.state, "unhealthy"); + assert.match(r.why, /never delivered/); + assert.equal(r.polls.found_unsaid, 3); + assert.equal(r.refused.count, 1); + assert.equal(r.refused.last.reason, "it is unsigned"); + assert.equal(r.hook.kind, "system"); + assert.equal(r.receiver.port, 3047); + // The first delivery clears it. + now += 1000; + pace.heard("push", "d-1"); + r = pace.report() as any; + assert.equal(r.healthy, true); + assert.equal(r.verified.count, 1); + assert.equal(r.verified.last.event, "push"); + assert.equal(r.polls.every_seconds, 300); + assert.match(said.join("\n"), /heard again/); +}); + +test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true, + did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } }; + pace.listening("172.17.0.1", 3047, hook.url, hook); + now += 60_000; + pace.heard("push", "d-1"); + // The hook is deleted by hand: nothing more comes, and the poll finds what no event said. + now += 11 * 60_000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event"); + pace.miss("the merge of novox/hq#7"); + pace.miss("novox/hq#8 at bbbbbbbb"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once"); + assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/); + assert.equal((pace.report() as any).healthy, false); + // A delivery ends the spell; the next spell is said again. + now += 1000; + pace.heard("push", "d-2"); + assert.equal((pace.report() as any).healthy, true); + now += 11 * 60_000; + pace.miss("novox/hq#9 at cccccccc"); + pace.miss("novox/hq#10 at dddddddd"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again"); + // Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once. + now += 1000; + pace.heard("push", "d-3"); + now += 1000; + pace.refused("its signature does not verify", "172.17.0.3"); + pace.refused("its signature does not verify", "172.17.0.3"); + const r = pace.report() as any; + assert.equal(r.healthy, false); + assert.equal(r.refused.since_verified, 2); + assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/); + now += 1000; + pace.heard("push", "d-4"); + assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it"); +}); + +test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const quiet = new Pace(() => now, (l) => said.push(l)); + quiet.unheard("MESH_GITEA_HOOK_URL unset", false); + now += 20 * 60_000; + quiet.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.length, 0, "no deliveries were expected"); + assert.equal((quiet.report() as any).healthy, true); + const failed = new Pace(() => now, () => {}); + failed.unheard("the forge's system webhook could not be made: 403"); + const r = failed.report() as any; + assert.equal(r.healthy, false); + assert.match(r.why, /403/); +}); + +test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + assert.equal(pace.every(), 30_000, "no delivery yet"); + pace.heard(); + assert.equal(pace.every(), 300_000, "a delivery heard"); + now += 11 * 60_000; + assert.equal(pace.every(), 30_000, "ten minutes without a delivery"); + pace.heard(); + now += 1000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(pace.every(), 30_000, "the poll found what no event said"); + assert.match(said.join("\n"), /MISSED novox\/hq#7/); + now += 1000; + pace.heard(); + assert.equal(pace.every(), 300_000, "the next delivery"); +}); diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 3aac30b1..8d7f7afd 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -12,7 +12,20 @@ // hands a token's value out exactly once, so a token not kept is a token lost; // - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not // a fault: the forge's data was restored from a predecessor and the token the file names never -// existed there. +// existed there; +// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process +// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two +// processes over one kept file. Two that minted at once left the forge two tokens by one name, which +// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14 +// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it +// holds it: what another process minted meanwhile is reused, not minted over; +// - replaced by id: an old token by this name is found in the account's list and deleted by its id, +// every one of them, so a name held twice is cleared rather than refused. +// +// - minted for the tools' account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's +// login mints it for `mesh-tools` (tools-account.ts), after making that account and its team, and deletes any +// token by this name the administrator itself still holds — the token the tools used before. A token kept +// for another account than the one asked (the scopes line says whose) is minted again at the next start. // // An explicitly configured token still wins, and is never minted over: if it is rejected, that is // reported, not repaired — somebody chose it. @@ -20,8 +33,10 @@ // The token is never logged. Lines say that one was minted, reused or renewed, and where it is // kept; never what it is. -import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; +import { setUpToolsAccount, TOOLS_ACCOUNT, type AdminCall } from "./tools-account.js"; /** The name the token carries in the forge's own list — one per mesh runtime, found by name. */ export const TOKEN_NAME = "mesh-tools"; @@ -39,7 +54,7 @@ export const TOKEN_NAME = "mesh-tools"; * `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the * operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token. * A token kept with a different list is minted again with this one at the next start (the scopes are kept - * beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this. + * beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this. * * A token kept from before a scope was added lacks it: the forge answers such a call with * `403 token does not have at least one of required scope(s)`, and the client treats that like a @@ -95,6 +110,14 @@ export interface MintedTokenOptions { readonly file: string; readonly name?: string; readonly scopes?: readonly string[]; + /** + * The account the token is minted for: the tools' own (novox/hq ADR 0365 §2). The admin login mints it, after + * `setUp` made the account, and the admin's own tokens by this name are deleted. Left out, the token is the + * admin's own, as it was before. + */ + readonly account?: string; + /** Run with the admin's password before each mint for `account`: makes the account and its team (tools-account.ts). */ + readonly setUp?: (password: string) => Promise; readonly log?: (line: string) => void; readonly fetch?: typeof fetch; } @@ -106,12 +129,14 @@ export class MintedToken implements TokenSource { private inflight: Promise | null = null; private readonly name: string; private readonly scopes: readonly string[]; + private readonly account: string; private readonly log: (line: string) => void; private readonly fetchImpl: typeof fetch; constructor(private readonly opts: MintedTokenOptions) { this.name = opts.name ?? TOKEN_NAME; this.scopes = opts.scopes ?? TOKEN_SCOPES; + this.account = opts.account ?? opts.admin; this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`)); this.fetchImpl = opts.fetch ?? fetch; } @@ -124,9 +149,9 @@ export class MintedToken implements TokenSource { */ static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken { const opts = MintedToken.optionsFromEnv(url, env); - // One source per kept file in a process. The watcher and the tools entrypoint both build a - // client in the same runtime; two sources over one file would each renew on a 401 and drop the - // other's token by name, forever. Shared, a renewal is one renewal. + // One source per kept file in a process: two sources in one process would each renew on a 401. + // This covers one process only. The watcher and the tools are two processes (the runtime starts + // each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below). const shared = MintedToken.shared.get(opts.file); if (shared) return shared; const source = new MintedToken(opts); @@ -148,7 +173,17 @@ export class MintedToken implements TokenSource { if (missing.length) { throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`); } - return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") }; + // The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it. + const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT; + if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`); + return { + url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token"), account, + // GiteaAdmin is loaded when first used: client.ts imports this file. + setUp: async (password) => { + const { GiteaAdmin } = await import("./client.js"); + await setUpToolsAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url")); + }, + }; } async current(): Promise { @@ -159,11 +194,13 @@ export class MintedToken implements TokenSource { if (kept !== null && this.keptScopes() === this.scopesLine()) { this.held = kept; this.log(`reusing the token kept at ${this.opts.file}`); + // A retirement of the admin's old token that failed after the mint is tried again, once a process. + if (existsSync(this.retireFile)) await this.locked(() => this.retireAdmin()).catch((err) => this.log(`${err}`)); return kept; } - if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them"); + if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept); } - return this.mint("no token kept — minting one"); + return this.mint("no token kept — minting one", null); } async renew(rejected: string): Promise { @@ -177,7 +214,7 @@ export class MintedToken implements TokenSource { return kept; } this.held = null; - return this.mint("the forge rejected the kept token — minting a fresh one"); + return this.mint("the forge rejected the kept token — minting a fresh one", rejected); } /** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */ @@ -185,17 +222,95 @@ export class MintedToken implements TokenSource { return status === 403 && /required scope/i.test(body); } - /** One mint at a time: concurrent first calls share it, rather than each minting its own. */ - private mint(why: string): Promise { + /** + * One mint at a time: concurrent first calls in this process share it, and a mint in another process + * on this machine is waited for (the lock file). Once the lock is held, the kept file is read again: + * a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the + * scopes asked for was minted by the process that held the lock before — it is reused. + */ + private mint(why: string, stale: string | null): Promise { if (this.inflight === null) { this.log(why); - this.inflight = this.doMint().finally(() => { + this.inflight = this.locked(async () => { + const kept = this.read(); + if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) { + this.held = kept; + this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`); + return kept; + } + return this.doMint(); + }).finally(() => { this.inflight = null; }); } return this.inflight; } + /** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */ + static readonly LOCK_STALE_MS = 120_000; + static readonly LOCK_WAIT_MS = 150_000; + + /** + * Run `work` holding the lock file beside the kept token: created exclusively, holding this process's + * pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone, + * or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never + * waited on for ever, which would be the same lock-out by another road. + */ + private async locked(work: () => Promise): Promise { + const lock = `${this.opts.file}.lock`; + mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 }); + const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`; + const deadline = Date.now() + MintedToken.LOCK_WAIT_MS; + for (;;) { + try { + const fd = openSync(lock, "wx", 0o600); + try { writeSync(fd, mine); } finally { closeSync(fd); } + break; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EEXIST") { + throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`); + } + } + if (this.leftBehind(lock)) { + this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`); + try { unlinkSync(lock); } catch { /* another waiter took it over first */ } + continue; + } + if (Date.now() > deadline) { + throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`); + } + await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50))); + } + try { + return await work(); + } finally { + try { + if (readFileSync(lock, "utf8") === mine) unlinkSync(lock); + } catch { /* already gone */ } + } + } + + /** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */ + private leftBehind(lock: string): boolean { + let text: string; + let age: number; + try { + text = readFileSync(lock, "utf8"); + age = Date.now() - statSync(lock).mtimeMs; + } catch { + return false; // released meanwhile: the next attempt takes it + } + if (age > MintedToken.LOCK_STALE_MS) return true; + const pid = Number(text.split(" ")[0]); + if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment + try { + process.kill(pid, 0); + return false; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "ESRCH"; + } + } + private read(): string | null { try { const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, ""); @@ -206,9 +321,10 @@ export class MintedToken implements TokenSource { } } - /** The scopes as kept beside the token: sorted, one line. */ + /** The scopes as kept beside the token: sorted, one line, and whose it is when it is not the admin's. */ private scopesLine(): string { - return [...this.scopes].sort().join(","); + const scopes = [...this.scopes].sort().join(","); + return this.account === this.opts.admin ? scopes : `${scopes} for ${this.account}`; } /** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */ @@ -231,17 +347,20 @@ export class MintedToken implements TokenSource { renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`); } - private async doMint(): Promise { - let password: string; + /** One call to the forge's API with the admin login, the password read now. */ + private password(): string { try { - password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, ""); + return readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, ""); } catch (err) { throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`); } + } + + private adminCall(password = this.password()): AdminCall { const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64"); - const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`; - const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => { - const res = await this.fetchImpl(tokens + path, { + const api = `${this.opts.url.replace(/\/+$/, "")}/api/v1`; + const admin: AdminCall = async (method, path, body) => { + const res = await this.fetchImpl(api + path, { method, headers: { "Content-Type": "application/json", Authorization: authorization }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), @@ -253,30 +372,97 @@ export class MintedToken implements TokenSource { } return { status: res.status, body: parsed }; }; + return admin; + } + + /** Where a retirement of the admin's old token that failed is remembered, to be tried again (retireAdmin). */ + private get retireFile(): string { + return `${this.opts.file}.retire-admin`; + } + + /** + * The admin's own token by this name goes: what the tools used before must not outlive the move (novox/hq ADR + * 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without + * a token: it is said, remembered beside the kept token, and tried again when a process next reads that token. + */ + private async retireAdmin(admin: AdminCall = this.adminCall()): Promise { + if (this.account === this.opts.admin) return; + try { + const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin); + if (retired > 0) this.log(`deleted ${retired} token(s) named "${this.name}" the admin account "${this.opts.admin}" held: the tools act as "${this.account}" now`); + rmSync(this.retireFile, { force: true }); + } catch (err) { + writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 }); + this.log(`FAILED to delete the token(s) named "${this.name}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`); + } + } + + private async doMint(): Promise { + const password = this.password(); + const admin = this.adminCall(password); + const tokensOf = (account: string) => `/users/${encodeURIComponent(account)}/tokens`; + const call = (method: string, path = "", body?: unknown) => admin(method, tokensOf(this.account) + path, body); + + if (this.account !== this.opts.admin) { + // The tools' account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again + // at the next call. + const me = await admin("GET", "/user"); + if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status); + await this.opts.setUp?.(password); + } let res = await call("POST", "", { name: this.name, scopes: this.scopes }); if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status); if (res.status === 400 || res.status === 422) { // The forge still holds a token by this name whose value we no longer have — the kept file - // went while the forge's data stayed. It is ours to replace: drop it by name and mint again. - this.log(`the forge already holds a token named "${this.name}" — replacing it`); - const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`); - if (dropped.status !== 204 && dropped.status !== 404) { - throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`); - } + // went while the forge's data stayed, or two processes once minted at once and left two. It is + // ours to replace: find every one by this name in the account's list and delete each by its id + // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint. + await this.dropNamed(admin, tokensOf(this.account), this.account, true); res = await call("POST", "", { name: this.name, scopes: this.scopes }); } if (res.status !== 201 && res.status !== 200) { - throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`); + throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} ${detail(res.body)}`); } const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null; - if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`); + if (!token) throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} but no token in the reply`); this.keep(token); this.held = token; - this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`); + this.log(`minted a token for "${this.account}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`); + // Only now, with the account's own token kept, does the admin's old one go. + await this.retireAdmin(admin); return token; } + + /** + * Delete every token by this name an account holds, each by its id (gitea refuses a delete by a name held twice: + * "multiple matches for token name"); answers how many. `replacing` says the line for a mint that found its name + * taken. + */ + private async dropNamed(admin: AdminCall, tokens: string, account: string, replacing = false): Promise { + const held: number[] = []; + for (let page = 1; page <= 100; page++) { + const listed = await admin("GET", `${tokens}?page=${page}&limit=50`); + if (listed.status !== 200 || !Array.isArray(listed.body)) { + throw new Error(`Gitea GET /users/${account}/tokens: ${listed.status} ${detail(listed.body)}`); + } + if (listed.body.length === 0) break; + for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id); + } + if (replacing) { + this.log(held.length === 1 + ? `the forge already holds a token named "${this.name}" — replacing it` + : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`); + } + for (const id of held) { + const dropped = await admin("DELETE", `${tokens}/${id}`); + if (dropped.status !== 204 && dropped.status !== 404) { + throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`); + } + } + return held.length; + } } function detail(body: unknown): string { diff --git a/modules/gitea/tools-account.ts b/modules/gitea/tools-account.ts new file mode 100644 index 00000000..03df8e32 --- /dev/null +++ b/modules/gitea/tools-account.ts @@ -0,0 +1,52 @@ +// The forge's tools act as an account of their own, not as the site administrator (novox/hq ADR 0365 §2, +// issue 469). +// +// The administrator's token reached everything the forge holds, so the tools' boundary rested on their own guards +// (guard.ts): a protection edit, say, the forge itself would have taken. Now the tools' token is minted for a plain +// account, `mesh-tools`, which is no administrator and writes to the mesh's repositories through one team of the +// mesh's organisation: +// +// - **the account**: made by the module with the administrator's login (that login sets the forge up and does +// nothing else), never an administrator — one found made an administrator is made a plain account again. Its +// password is random, sent once and kept nowhere: the account is reached only by the tokens the administrator +// mints for it (gitea lets an administrator mint for another account: `reqSelfOrAdmin`). +// - **the team**: `mesh-tools` in the organisation `novox`, on every repository of it, writing code (pushes to a +// branch, commit statuses, merges), issues (notes, comments, labels) and pull requests. No administration of a +// repository, so the forge refuses the account a protection edit, a repository's settings and its deletion; +// no repository made in the organisation, since its maker would be that repository's administrator. +// +// Every step is idempotent (GiteaAdmin's, client.ts), and run with the administrator's login before each mint of the tools' token, so a team +// or a membership lost behind the module's back is made again at the next mint. + +/** The account the tools act as. */ +export const TOOLS_ACCOUNT = "mesh-tools"; +/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */ +export const TOOLS_ORG = "novox"; +export const TOOLS_TEAM = "mesh-tools"; + +/** What the team grants: write on code, issues and pull requests of every repository of the organisation, and no more. */ +export const TOOLS_TEAM_UNITS = { + permission: "write", + units_map: { "repo.code": "write", "repo.issues": "write", "repo.pulls": "write" }, + includes_all_repositories: true, + can_create_org_repo: false, +} as const; + +/** One call to the forge's API with the administrator's login: a path under /api/v1, the status and body answered. */ +export type AdminCall = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: any }>; + +/** What setting the account up needs of the forge's admin API: GiteaAdmin's own (client.ts), shared with the + * package provisioner, so a create race (422) and a user made by hand are handled once. */ +export interface ToolsForge { + ensureUser(username: string, password: string, email: string, plain?: boolean): Promise; + ensureTeamWith(org: string, team: string, units: Record): Promise; + addUserToTeam(teamId: number, username: string): Promise; +} + +/** Make the tools' account, plain, a member of its team on every repository of the organisation. Its password is + * set afresh each time and kept nowhere: the account is reached only by the tokens the administrator mints. */ +export async function setUpToolsAccount(forge: ToolsForge, account = TOOLS_ACCOUNT, password: () => string): Promise { + await forge.ensureUser(account, password(), `${account}@noreply.localhost`, true); + const team = await forge.ensureTeamWith(TOOLS_ORG, TOOLS_TEAM, { name: TOOLS_TEAM, ...TOOLS_TEAM_UNITS }); + await forge.addUserToTeam(team, account); +} diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index b8c46151..27a8c610 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -10,7 +10,7 @@ import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; import { emit } from "@novox/mesh-sdk/events"; -import { GiteaClient, readyAuthors, type BranchProtection } from "../client.js"; +import { GiteaAdmin, GiteaClient, readyAuthors, type BranchProtection } from "../client.js"; import { appendNote, run, viewBody, viewComment } from "../delivery.js"; import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js"; import { @@ -59,13 +59,13 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] { })); } -export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] { - return namesChecked(allGiteaTools(gitea)); +export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { + return namesChecked(allGiteaTools(gitea, admin)); } // gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one // writer (novox/hq issue 456, ADR 0323). -function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { +function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { return [ // ---- Repositories ---- { @@ -401,8 +401,11 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { }, run: async (args) => { const owner = String(args.owner), repo = String(args.repo); - if (!args.branch) return { rules: (await gitea.branchProtections(owner, repo)).map(summarised) }; - const rule = await gitea.branchProtection(owner, repo, String(args.branch)); + // The one read made as the forge's administrator (novox/hq ADR 0375, issue 772): the rules' list is the + // repository administrator's, which the tools' account is not. No other tool uses this account. + const rules = await (admin ? admin() : gitea).branchProtections(owner, repo); + if (!args.branch) return { rules: rules.map(summarised) }; + const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === String(args.branch)) ?? null; return { branch: String(args.branch), rule: rule ? summarised(rule) : null }; }, }, @@ -504,7 +507,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { base = base || "main"; let rule: BranchProtection | null = null, unreadable: string | undefined; try { - rule = await gitea.branchProtection(owner, repo, base); + rule = await gitea.appliedProtection(owner, repo, base); } catch (err) { unreadable = err instanceof Error ? err.message : String(err); } @@ -589,8 +592,12 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { const { owner, repo, branch, newBranch } = judged.contents; const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`); const defaultBranch = String(found?.default_branch ?? "main"); - const refused = judgeContentsWrite(newBranch ?? branch ?? defaultBranch, defaultBranch, - await gitea.branchProtections(owner, repo)); + // The protection the forge applies to the branch written, read from the branch: the rules' own list is a + // repository administrator's, which the tools' account is not (novox/hq ADR 0365). A branch not made yet is + // left to the forge, which refuses a write that would make a branch a rule guards. + const target = newBranch ?? branch ?? defaultBranch; + const applied = target === defaultBranch ? null : await gitea.appliedProtection(owner, repo, target); + const refused = judgeContentsWrite(target, defaultBranch, applied ? [applied] : []); if (refused) throw new Refused(refused); } // The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts). @@ -611,7 +618,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { // way, gitea contributes none rather than failing the whole runtime, and says why. registerModuleTools("gitea", (env) => { try { - return getGiteaTools(GiteaClient.fromEnv(env)); + return getGiteaTools(GiteaClient.fromEnv(env), () => GiteaAdmin.fromEnv(env)); } catch (err) { console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`); return []; diff --git a/modules/gitea/webhook.ts b/modules/gitea/webhook.ts new file mode 100644 index 00000000..7cb63671 --- /dev/null +++ b/modules/gitea/webhook.ts @@ -0,0 +1,519 @@ +// **The forge says every event on the bus** (novox/hq issue 729; to-be 10: an event makes it fast, nothing makes it +// necessary). The module keeps one system webhook on its forge for every event, pointing at its own receiver, signed +// with a secret it mints and keeps beside its token; the receiver refuses whatever is unsigned or does not verify, and +// publishes each event, trimmed, on the module's events as forge.. (forge.pull-request.opened, forge.push, …), +// on the shared EVENTS path its other events take; its own stream (ADR 0363) is a follow-up once issue 725 ships. +// The polls stay as the safety net, slowed; the announced-heads and announced-merges records keep anything from +// being announced twice, whichever path saw it first. +// +// The secret is never logged: lines say it was minted or reused and where it is kept, never what it is. + +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; +import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import { dirname } from "node:path"; + +/** The name the receiver answers on, and the port it listens on unless the manifest says otherwise. */ +export const HOOK_PATH = "/forge-event"; +export const HOOK_PORT = 3047; + +/** The secret the forge signs with: read from its 0600 file, or minted and kept there. */ +export function keptSecret(file: string, log: (line: string) => void = (l) => console.log(`[gitea] ${l}`)): string { + try { + const kept = readFileSync(file, "utf8").trim(); + if (kept.length >= 32) { + log(`reusing the webhook secret kept at ${file}`); + return kept; + } + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + } + const secret = randomBytes(32).toString("hex"); + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + const tmp = `${file}.tmp`; + writeFileSync(tmp, secret + "\n", { mode: 0o600 }); + chmodSync(tmp, 0o600); + renameSync(tmp, file); + log(`minted a webhook secret, kept at ${file}`); + return secret; +} + +/** Whether a body carries the forge's HMAC-SHA256 signature (X-Gitea-Signature, hex) under the secret. */ +export function verified(secret: string, body: Buffer | string, signature: string | undefined): boolean { + if (!secret || !signature || !/^[0-9a-f]{64}$/i.test(signature.trim())) return false; + const want = createHmac("sha256", secret).update(body).digest(); + const got = Buffer.from(signature.trim(), "hex"); + return got.length === want.length && timingSafeEqual(got, want); +} + +/** One forge event as the bus carries it: its name inside the stream, and what is needed of it — no secrets. */ +export interface ForgeEvent { + name: string; + body: Record; +} + +/** Every event name the module emits for the forge, without its forge. prefix (module.json `emits`). */ +export const EVENTS: readonly string[] = ["push", "create", "delete", "fork", "status", "pull-request.opened", "pull-request.reopened", "pull-request.synchronized", "pull-request.edited", "pull-request.closed", "pull-request.merged", "pull-request.labeled", "pull-request.unlabeled", "pull-request.assigned", "pull-request.unassigned", "pull-request.reviewed", "pull-request.review-requested", "pull-request.commented", "pull-request.changed", "issue-comment.created", "issue-comment.edited", "issue-comment.deleted", "issues.opened", "issues.reopened", "issues.closed", "issues.edited", "issues.labeled", "issues.unlabeled", "issues.assigned", "issues.unassigned", "issues.changed", "release.published", "release.updated", "release.deleted", "release.changed", "repository.created", "repository.deleted", "repository.changed", "wiki.changed", "other"]; + +/** The families the module's stream carries (module.json `streams`); any other kind is said as other.; underscores become hyphens. */ +export const FAMILIES = ["push", "create", "delete", "fork", "status", "pull_request", "issue_comment", "issues", + "release", "repository", "wiki"] as const; + +const ACTIONS: Record = { label_updated: "labeled", label_cleared: "unlabeled", synchronized: "synchronized" }; + +function str(v: unknown): string | undefined { + return typeof v === "string" && v ? v : undefined; +} +function num(v: unknown): number | undefined { + return typeof v === "number" ? v : undefined; +} +function login(u: any): string | undefined { + return str(u?.login) ?? str(u?.username); +} + +/** + * The event a forge delivery is: its kind (X-Gitea-Event) and the payload's action, trimmed to ids, repository, refs, + * shas, author, labels and a comment's body (a ready mark is one, ADR 0364). A merge is pull-request.merged, a comment on + * a pull request pull-request.commented. + */ +export function eventOf(kind: string, payload: any): ForgeEvent { + // The bus's event names take lower-case letters, digits and hyphens: the forge's underscores become hyphens. + const event = forgeEventOf(kind, payload); + const name = event.name.toLowerCase().replace(/_/g, "-").replace(/[^a-z0-9.-]/g, "-"); + // Published on the module's events as it publishes its others, each name declared in its `emits`: one the manifest + // does not name is forge.other, saying what the forge called it. Its own stream (ADR 0363) follows issue 725. + if (EVENTS.includes(name)) return { name: `forge.${name}`, body: event.body }; + return { name: "forge.other", body: { ...event.body, kind, said: name } }; +} + +function forgeEventOf(said: string, payload: any): ForgeEvent { + // The forge names some kinds finer than their family (pull_request_label, pull_request_sync): read by the payload. + let kind = said; + if (said.startsWith("pull_request") && payload?.pull_request) kind = "pull_request"; + if ((said.startsWith("issue") || said === "pull_request_comment") && payload?.comment) kind = "issue_comment"; + if (said.startsWith("issue") && !payload?.comment && payload?.issue) kind = "issues"; + const repo = payload?.repository ?? {}; + const base: Record = { + owner: login(repo.owner), repo: str(repo.name), full_name: str(repo.full_name), clone_url: str(repo.clone_url), + sender: login(payload?.sender), + }; + const family = (FAMILIES as readonly string[]).includes(kind) ? kind : `other`; + let action = str(payload?.action); + const pull = payload?.pull_request; + if (kind === "push") { + return { name: "push", body: { ...base, ref: str(payload.ref), before: str(payload.before), after: str(payload.after), + commits: Array.isArray(payload.commits) ? payload.commits.length : undefined, pusher: login(payload.pusher) } }; + } + if (kind === "pull_request" && pull) { + if (action === "closed" && pull.merged) action = "merged"; + action = ACTIONS[action ?? ""] ?? action ?? "changed"; + return { name: `pull_request.${action}`, body: { ...base, number: num(pull.number ?? payload.number), + title: str(pull.title), author: login(pull.user), head: str(pull.head?.ref), head_sha: str(pull.head?.sha), + base: str(pull.base?.ref), merged: !!pull.merged, merge_commit_sha: str(pull.merge_commit_sha), + merged_at: str(pull.merged_at), html_url: str(pull.html_url), + labels: Array.isArray(pull.labels) ? pull.labels.map((l: any) => str(l?.name)).filter(Boolean) : undefined, + label: str(payload.label?.name) } }; + } + if (kind === "issue_comment") { + const issue = payload.issue ?? {}; + const onPull = !!payload.is_pull || !!issue.pull_request; + return { name: onPull ? "pull_request.commented" : `issue_comment.${action ?? "created"}`, body: { ...base, + number: num(issue.number), comment_id: num(payload.comment?.id), author: login(payload.comment?.user), + comment: str(payload.comment?.body), action } }; + } + if (kind === "create" || kind === "delete") { + return { name: kind, body: { ...base, ref: str(payload.ref), ref_type: str(payload.ref_type), sha: str(payload.sha) } }; + } + if (kind === "status") { + return { name: "status", body: { ...base, sha: str(payload.sha), context: str(payload.context), state: str(payload.state) } }; + } + const name = family === "other" ? `other.${kind.replace(/[^a-z0-9_-]/gi, "_") || "unknown"}` : + ["fork"].includes(family) ? family : `${family}.${action ?? "changed"}`; + return { name, body: { ...base, number: num(payload?.number ?? payload?.issue?.number), action } }; +} + +export interface ReceiverOptions { + secret: string; + port: number; + host?: string; + /** Publishes one event on the bus. */ + publish: (event: ForgeEvent) => Promise; + /** Acts on it at once, beside publishing it (the announcer). */ + act?: (event: ForgeEvent) => Promise; + /** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are + * heard, and the module can say when it last heard one. */ + heard?: (kind: string, delivery: string) => void; + /** Told of every refused delivery, with the reason and where it came from. */ + refused?: (reason: string, from: string) => void; + log?: (line: string) => void; +} + +/** The most a delivery may carry: the forge's push of many commits stays well under it. */ +export const MAX_BODY = 5 * 1024 * 1024; +/** How many deliveries' ids are remembered, so a delivery sent again is published once. */ +export const REMEMBERED = 2000; + +/** + * The receiver. A request without a signature is refused before its body is read, a body over MAX_BODY is refused, and + * one whose signature does not verify is refused, each with 401 or 413 and a line saying so. A verified delivery is + * answered 200 at once and queued: it is published and acted on after the reply, one at a time, so the forge's + * delivery timeout is never reached. A delivery whose X-Gitea-Delivery id was already heard is answered and dropped. + */ +export function receiver(opts: ReceiverOptions): Server & { idle(): Promise } { + const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`)); + const refuse = (from: string, reason: string): void => { + log(`refused a forge event from ${from}: ${reason}`); + opts.refused?.(reason, from); + }; + const seen = new Set(); + let queue: Promise = Promise.resolve(); + const server = createServer((req, res) => { + if (req.method !== "POST" || (req.url ?? "").split("?")[0] !== HOOK_PATH) { + res.statusCode = 404; + return res.end(); + } + const from = req.socket.remoteAddress ?? "?"; + const signature = String(req.headers["x-gitea-signature"] ?? "").trim(); + if (!signature) { + refuse(from, "it is unsigned"); + res.writeHead(401, { Connection: "close" }); + res.end("unsigned\n"); + req.destroy(); + return; + } + if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) { + refuse(from, `its body is over ${MAX_BODY} bytes`); + res.writeHead(413, { Connection: "close" }); + res.end("too large\n"); + req.destroy(); + return; + } + const chunks: Buffer[] = []; + let size = 0; + let over = false; + req.on("data", (c: Buffer) => { + size += c.length; + if (size > MAX_BODY) { + if (!over) { + over = true; + refuse(from, `its body is over ${MAX_BODY} bytes`); + res.writeHead(413, { Connection: "close" }); + res.end("too large\n"); + req.destroy(); + } + } else chunks.push(c); + }); + req.on("end", () => { + if (over) return; + const body = Buffer.concat(chunks); + if (!verified(opts.secret, body, signature)) { + refuse(from, "its signature does not verify"); + res.statusCode = 401; + return res.end("signature does not verify\n"); + } + let payload: unknown; + try { + payload = JSON.parse(body.toString("utf8")); + } catch { + refuse(from, "it is signed but not JSON"); + res.statusCode = 400; + return res.end("not JSON\n"); + } + const delivery = String(req.headers["x-gitea-delivery"] ?? ""); + const kind = String(req.headers["x-gitea-event"] ?? "other"); + opts.heard?.(kind, delivery); + if (delivery && seen.has(delivery)) { + res.statusCode = 200; + return res.end("already heard\n"); + } + if (delivery) { + seen.add(delivery); + if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string); + } + const event = eventOf(kind, payload); + res.statusCode = 200; + res.end("queued\n"); + queue = queue.then(async () => { + try { + await opts.publish(event); + if (opts.act) await opts.act(event); + } catch (err) { + log(`a forge event ${event.name} could not be published or acted on; the polls catch it — ${err instanceof Error ? err.message : err}`); + } + }); + }); + }); + return Object.assign(server, { idle: () => queue }); +} + +/** What ensuring the system webhook needs of the forge's administration (GiteaAdmin's login). */ +export interface HookAdmin { + call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>; +} + +/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */ +export interface HookState { + /** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */ + kind: "system"; + id: number; + url: string; + events: string[]; + active: boolean; + content_type?: string; + /** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */ + did: "created" | "replaced"; + /** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which + * would have sent that repository's events a second time. */ + removed: { system: number; default: number; repository: number; repositories: string[] }; +} + +/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll + * finds what no event said. */ +export const SILENT_AFTER = 10 * 60_000; + +/** + * **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the + * forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first + * delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds. + * + * It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused + * deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver + * listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery — + * and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an + * event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a + * delivery verifies. Deliveries refused since the last verified one are said unhealthy too. + */ +export class Pace { + private last = 0; + private missed = 0; + private readonly started: number; + private readonly now: () => number; + private readonly log: (l: string) => void; + private where: { host: string; port: number; url: string } | null = null; + private hook: HookState | null = null; + private notHeard: string | null = null; + private notHeardFails = false; + private verifiedCount = 0; + private lastVerified: { at: number; event: string; delivery: string } | null = null; + private refusedCount = 0; + private refusedSpell = 0; + private lastRefused: { at: number; reason: string; from: string } | null = null; + private missCount = 0; + private lastMiss: { at: number; what: string } | null = null; + private silentSince = 0; + private quietMisses = 0; + constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) { + this.now = now; + this.log = log; + this.started = now(); + } + /** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */ + listening(host: string, port: number, url: string, hook: HookState): void { + this.where = { host, port, url }; + this.hook = hook; + this.notHeard = null; + } + /** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen + * or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */ + unheard(why: string, fails = true): void { + this.notHeard = why; + this.notHeardFails = fails; + } + heard(event = "", delivery = ""): void { + this.last = this.now(); + this.verifiedCount++; + this.lastVerified = { at: this.last, event, delivery }; + this.refusedSpell = 0; + this.quietMisses = 0; + if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`); + this.silentSince = 0; + } + refused(reason: string, from: string): void { + this.refusedCount++; + this.refusedSpell++; + this.lastRefused = { at: this.now(), reason, from }; + } + /** The poll found a head or merge no forge event announced. */ + miss(what: string): void { + const t = this.now(); + if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`); + this.missed = t; + this.missCount++; + this.lastMiss = { at: t, what }; + // Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by + // hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell. + const quietSince = this.last || this.started; + this.quietMisses++; + if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) { + this.silentSince = t; + const minutes = Math.round((t - quietSince) / 60_000); + const refusedSince = this.refusedSinceHeard(); + this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` + + `${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` + + `system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` + + `${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` + + `(novox/hq issue 729)`); + } + } + /** Whether a delivery was refused after the last verified one (or with none verified). */ + private refusedSinceHeard(): number { + return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0; + } + hearing(): boolean { + const t = this.now(); + return this.last > 0 && t - this.last < 10 * 60_000 && this.last > this.missed; + } + every(): number { + return this.hearing() ? 5 * 60_000 : 30_000; + } + private expecting(): boolean { + return this.where !== null && this.hook !== null; + } + /** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */ + report(): Record { + const t = this.now(); + const iso = (ms: number): string => new Date(ms).toISOString(); + let state: "healthy" | "unhealthy" | "waits" = "healthy"; + let why: string; + const refusedSince = this.refusedSinceHeard(); + if (this.silentSince) { + state = "unhealthy"; + why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` + + `event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` + + `event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`; + } else if (refusedSince) { + state = "unhealthy"; + why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` : + "and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`; + } else if (!this.expecting()) { + why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` : + "the receiver is not listening yet"; + if (this.notHeard && this.notHeardFails) state = "unhealthy"; + } else if (this.hearing()) { + why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`; + } else if (this.last === 0) { + why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`; + } else { + why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`; + } + return { + healthy: state !== "unhealthy", state, why, + started: iso(this.started), + receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" }, + hook: this.hook, + verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } }, + refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } }, + polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount, + last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } }, + silent: this.silentSince ? { since: iso(this.silentSince) } : null, + }; + } +} + +/** Every event family the forge (1.27) can send a webhook. */ +export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request", + "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"]; + +async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise { + const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`); + if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`); + return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url); +} + +/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */ +const PAGE = 50; +async function everyPage(admin: HookAdmin, path: string, what: string): Promise { + const all: any[] = []; + for (let page = 1; ; page++) { + const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`); + if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`); + const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : []; + all.push(...items); + if (items.length < PAGE) return all; + } +} + +/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */ +async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> { + const found: { repo: string; id: number }[] = []; + for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) { + const full = String(repo?.full_name ?? ""); + if (!full.includes("/")) continue; + const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`); + for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) }); + } + return found; +} + +/** + * **One system webhook, for every event, at the receiver** (issue 729), made each start and read back. + * + * The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless + * `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories + * made after it, and the listing of system webhooks does not name it — so without the flag every start made another + * one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is + * not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a + * gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a + * default one, which would send that repository's events twice. The forge's listing of system webhooks must then name + * exactly one at the url, or this fails, saying so. + */ +export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise { + // The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. + const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", + config: { url, content_type: "json", secret, is_system_webhook: "true" } }; + const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id))); + const made = await admin.call("POST", "/admin/hooks", want); + if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`); + const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id))); + if (fresh.length !== 1) { + throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` + + `are not known to be sent`); + } + const ours = fresh[0]; + const defaults = await hooksAt(admin, "default", url); + const gone = async (path: string): Promise => { + const r = await admin.call("DELETE", path); + if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`); + }; + for (const id of before) await gone(`/admin/hooks/${id}`); + for (const h of defaults) await gone(`/admin/hooks/${h.id}`); + const copies = await repoHooksAt(admin, url); + for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`); + const kept = await hooksAt(admin, "system", url); + if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) { + throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` + + `made (${ours.id})`); + } + const removed = { system: before.size, default: defaults.length, repository: copies.length, + repositories: [...new Set(copies.map((c) => c.repo))] }; + return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [], + active: !!ours.active, content_type: ours.config?.content_type, + did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed }; +} + +/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */ +export function eventsTool(pace: Pace): { name: string; description: string; input: Record; + run: (args: Readonly>) => Promise } { + return { + name: "gitea_events", + description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " + + "system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " + + "verified and the last one's time and event, how many were refused and the last reason, how often the polls look " + + "and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " + + "forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " + + "poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " + + "could not listen or the webhook could not be made.", + input: {}, + run: async () => pace.report(), + }; +} + +/** + * **Claim a head for announcing, once** (issue 729): true when `sha` is not already the head recorded for `key`, which + * it then is. The poll and the forge's event both claim before they announce, so whichever comes second finds it taken. + */ +export function claimHead(heads: Record, key: string, sha: string): boolean { + if (!sha || heads[key] === sha) return false; + heads[key] = sha; + return true; +} diff --git a/modules/kernel/README.md b/modules/kernel/README.md new file mode 100644 index 00000000..dddcebd4 --- /dev/null +++ b/modules/kernel/README.md @@ -0,0 +1,52 @@ +# kernel + +The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one: +**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot +entries) is not built yet. + +## Why (novox/hq issue 746) + +With transparent huge pages `always`, the page cache of a file mapped by a program is collapsed into +2 MiB folios (khugepaged), and a small write into such a folio dirties all of it: writeback then writes +2 MiB. journald writes its journal files through mmap, and every journal reader (fail2ban's journal +jails among them) maps them too. Measured on the control node on 2026-10-14: journald's cgroup had +written 2.5 TB in 12 days while the journal grew by about 25-30 KB/s, fail2ban's cgroup 63 GB in 4 h +while its own files did not grow, both holding hundreds of MiB of file pages in huge folios; the bursts +saturated the mirrored spinning root disk that holds the bus store and the store, and the controller +lost its lease four times in 30 minutes. With `madvise`, only memory a program asks to have in huge pages +(`madvise(MADV_HUGEPAGE)`) gets them; a database or a JVM that wants them still asks. + +## What it owns + +| | what | notes | +|---|---|---| +| setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `always`, what the distribution's kernel boots with, so assigning the module changes nothing until a machine sets another value | +| file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` | +| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); reloading it runs `apply`, stopping it runs `restore`, which also removes `found`. Sandboxed: the file system read-only but for the module's state (`/sys` stays writable), no home, no network, no new privileges; the journal's rotation goes over its local socket, which a private network leaves reachable | +| service | `mesh-kernel-huge-pages.service` | running and enabled; **reloaded** when the script (so the setting) changes, so a new value is set directly and never passes back through the found one; restarted only when the unit changes; its health is the unit's | + +The per-size controls (`hugepages-kB/enabled`) are left alone: on the machines read, 2048 kB says +`inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one. + +**A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed +from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates +the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped +under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. No +rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing` +or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit. + +## Tools + +| tool | | answers | +|---|---|---| +| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) | + +## Tests + +`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the +manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh +writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, +rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a +reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel +does not take before writing anything, fails when the write does not take, and says when the kernel has +no transparent huge pages. diff --git a/modules/kernel/cmd/kernel-tools/hugepages.go b/modules/kernel/cmd/kernel-tools/hugepages.go new file mode 100644 index 00000000..697f4fd8 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/hugepages.go @@ -0,0 +1,241 @@ +package main + +import ( + "fmt" + "os" + "path" + "sort" + "strconv" + "strings" +) + +// THPDir is where the kernel says and takes its transparent huge pages. +const THPDir = "sys/kernel/mm/transparent_hugepage" + +// CgroupRoot is the unified cgroup hierarchy. +const CgroupRoot = "sys/fs/cgroup" + +// HugePagesAnswer is what kernel_huge_pages answers. +type HugePagesAnswer struct { + // Mode is the word the kernel brackets in enabled: always, madvise or never. + Mode string `json:"mode"` + Defrag string `json:"defrag,omitempty"` + Shmem string `json:"shmem_enabled,omitempty"` + PerSize []Size `json:"per_size,omitempty"` + // Meminfo is the machine's huge pages by kind, in MiB. + Meminfo map[string]float64 `json:"meminfo_mib,omitempty"` + Units []Unit `json:"units"` + Said []string `json:"said"` +} + +// Size is one per-size control (hugepages-kB/enabled). +type Size struct { + Size string `json:"size"` + Enabled string `json:"enabled"` + // Means is what inherit resolves to: the top-level mode. + Means string `json:"means"` +} + +// Unit is one cgroup holding file pages in huge folios. +type Unit struct { + Cgroup string `json:"cgroup"` + FileTHP int64 `json:"file_thp_bytes"` + FileDirty int64 `json:"file_dirty_bytes"` + FileWriteback int64 `json:"file_writeback_bytes"` + FileMapped int64 `json:"file_mapped_bytes"` + CollapsedByKhug int64 `json:"thp_collapse_alloc"` + Written int64 `json:"written_bytes"` +} + +// bracketed is the word a sysfs choice file brackets: "always [madvise] never" is madvise. +func bracketed(text string) string { + open := strings.IndexByte(text, '[') + end := strings.IndexByte(text, ']') + if open < 0 || end < open { + return "" + } + return text[open+1 : end] +} + +// HugePages reads the mode, the per-size controls, meminfo and the top units by file_thp. +func (m *Machine) HugePages(units int) (*HugePagesAnswer, error) { + enabled, err := m.read(THPDir + "/enabled") + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("this kernel has no /%s/enabled: it was built without transparent huge pages", THPDir) + } + return nil, fmt.Errorf("reading /%s/enabled: %w", THPDir, err) + } + a := &HugePagesAnswer{Mode: bracketed(enabled), Units: []Unit{}} + if s, err := m.read(THPDir + "/defrag"); err == nil { + a.Defrag = bracketed(s) + } + if s, err := m.read(THPDir + "/shmem_enabled"); err == nil { + a.Shmem = bracketed(s) + } + a.Said = append(a.Said, fmt.Sprintf("transparent huge pages: %s, defrag %s", a.Mode, orUnknown(a.Defrag))) + + sizes, _ := m.dirs(THPDir) + for _, d := range sizes { + if !strings.HasPrefix(d, "hugepages-") { + continue + } + s, err := m.read(THPDir + "/" + d + "/enabled") + if err != nil { + continue + } + sz := Size{Size: strings.TrimPrefix(d, "hugepages-"), Enabled: bracketed(s)} + sz.Means = sz.Enabled + if sz.Enabled == "inherit" { + sz.Means = a.Mode + } + a.PerSize = append(a.PerSize, sz) + } + sort.Slice(a.PerSize, func(i, j int) bool { return kib(a.PerSize[i].Size) < kib(a.PerSize[j].Size) }) + var on []string + for _, s := range a.PerSize { + if s.Means != "never" { + on = append(on, s.Size+" "+s.Means) + } + } + if len(a.PerSize) > 0 { + a.Said = append(a.Said, "sizes not never: "+orNone(on)) + } + + if mi, err := m.read("proc/meminfo"); err == nil { + a.Meminfo = meminfo(mi) + a.Said = append(a.Said, fmt.Sprintf("huge pages held: anonymous %.0f MiB, shared memory %.0f MiB, file %.0f MiB (mapped at 2 MiB %.0f MiB)", + a.Meminfo["AnonHugePages"], a.Meminfo["ShmemHugePages"], a.Meminfo["FileHugePages"], a.Meminfo["FilePmdMapped"])) + } + + if units > 0 { + all := m.cgroups() + sort.Slice(all, func(i, j int) bool { + if all[i].FileTHP != all[j].FileTHP { + return all[i].FileTHP > all[j].FileTHP + } + return all[i].Cgroup < all[j].Cgroup + }) + if len(all) > units { + all = all[:units] + } + a.Units = all + for _, u := range all { + a.Said = append(a.Said, fmt.Sprintf("%s: file in huge folios %s, dirty %s, writeback %s, written %s since it started", + u.Cgroup, mib(u.FileTHP), mib(u.FileDirty), mib(u.FileWriteback), mib(u.Written))) + } + if len(all) == 0 { + a.Said = append(a.Said, "no unit holds file pages in huge folios") + } + } + return a, nil +} + +// cgroups walks the leaf services and scopes (to three levels below the root) that hold file_thp. +func (m *Machine) cgroups() []Unit { + var out []Unit + var walk func(rel string, depth int) + walk = func(rel string, depth int) { + children, _ := m.dirs(path.Join(CgroupRoot, rel)) + for _, c := range children { + p := path.Join(rel, c) + if strings.HasSuffix(c, ".slice") { + if depth < 3 { + walk(p, depth+1) + } + continue + } + if !strings.HasSuffix(c, ".service") && !strings.HasSuffix(c, ".scope") { + continue + } + stat, err := m.read(path.Join(CgroupRoot, p, "memory.stat")) + if err != nil { + continue + } + kv := memoryStat(stat) + if kv["file_thp"] == 0 { + continue + } + u := Unit{Cgroup: p, FileTHP: kv["file_thp"], FileDirty: kv["file_dirty"], FileWriteback: kv["file_writeback"], + FileMapped: kv["file_mapped"], CollapsedByKhug: kv["thp_collapse_alloc"]} + if io, err := m.read(path.Join(CgroupRoot, p, "io.stat")); err == nil { + u.Written = mostWritten(io) + } + out = append(out, u) + } + } + walk("", 0) + return out +} + +// memoryStat reads memory.stat's "key value" lines. +func memoryStat(text string) map[string]int64 { + kv := map[string]int64{} + for _, l := range strings.Split(text, "\n") { + f := strings.Fields(l) + if len(f) != 2 { + continue + } + if n, err := strconv.ParseInt(f[1], 10, 64); err == nil { + kv[f[0]] = n + } + } + return kv +} + +// mostWritten is the most bytes any one device of io.stat saw written: a mirror's member disks and the +// mirror itself each count the same write, so the lines are never summed. +func mostWritten(text string) int64 { + var most int64 + for _, l := range strings.Split(text, "\n") { + for _, f := range strings.Fields(l) { + if v, ok := strings.CutPrefix(f, "wbytes="); ok { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > most { + most = n + } + } + } + } + return most +} + +// meminfo reads the huge page lines of /proc/meminfo, in MiB. +func meminfo(text string) map[string]float64 { + want := map[string]bool{"AnonHugePages": true, "ShmemHugePages": true, "ShmemPmdMapped": true, "FileHugePages": true, "FilePmdMapped": true} + out := map[string]float64{} + for _, l := range strings.Split(text, "\n") { + k, v, ok := strings.Cut(l, ":") + if !ok || !want[k] { + continue + } + f := strings.Fields(v) + if len(f) == 0 { + continue + } + if n, err := strconv.ParseFloat(f[0], 64); err == nil { + out[k] = n / 1024 + } + } + return out +} + +func kib(size string) int { + n, _ := strconv.Atoi(strings.TrimSuffix(size, "kB")) + return n +} + +func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) } + +func orUnknown(s string) string { + if s == "" { + return "unknown" + } + return s +} + +func orNone(list []string) string { + if len(list) == 0 { + return "none" + } + return strings.Join(list, ", ") +} diff --git a/modules/kernel/cmd/kernel-tools/hugepages_test.go b/modules/kernel/cmd/kernel-tools/hugepages_test.go new file mode 100644 index 00000000..2d6359c7 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/hugepages_test.go @@ -0,0 +1,113 @@ +package main + +// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for +// novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and +// journald's memory.stat and io.stat, cut to the lines the tool reads. + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func tree(t *testing.T, files map[string]string) *Machine { + t.Helper() + root := t.TempDir() + for p, content := range files { + full := filepath.Join(root, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return &Machine{Root: root} +} + +func novox(t *testing.T) *Machine { + thp := "sys/kernel/mm/transparent_hugepage/" + cg := "sys/fs/cgroup/system.slice/" + files := map[string]string{ + thp + "enabled": "[always] madvise never\n", + thp + "defrag": "always defer defer+madvise [madvise] never\n", + thp + "shmem_enabled": "always within_size [advise] never deny force\n", + thp + "hugepages-2048kB/enabled": "always [inherit] madvise never\n", + thp + "hugepages-64kB/enabled": "always inherit madvise [never]\n", + thp + "hugepages-1024kB/enabled": "always inherit madvise [never]\n", + "proc/meminfo": "MemTotal: 65748808 kB\nAnonHugePages: 3579904 kB\nShmemHugePages: 4096 kB\n" + + "ShmemPmdMapped: 4096 kB\nFileHugePages: 1693696 kB\nFilePmdMapped: 882688 kB\nHugepagesize: 2048 kB\n", + cg + "fail2ban.service/memory.stat": "anon 21815296\nfile 204623872\nfile_mapped 130736128\nfile_dirty 27262976\n" + + "file_writeback 4096\nanon_thp 0\nfile_thp 197132288\nthp_fault_alloc 0\nthp_collapse_alloc 550\n", + cg + "fail2ban.service/io.stat": "9:0 \n8:16 rbytes=3223552 wbytes=67723415552 rios=240 wios=281159 dbytes=0 dios=0\n" + + "8:0 rbytes=6152192 wbytes=67723415552 rios=832 wios=281159 dbytes=0 dios=0\n" + + "9:2 rbytes=9375744 wbytes=67723415552 rios=1072 wios=281159 dbytes=0 dios=0\n", + cg + "systemd-journald.service/memory.stat": "anon 3227648\nfile 1032015872\nfile_mapped 131928064\nfile_dirty 622592\n" + + "file_writeback 159744\nanon_thp 2097152\nfile_thp 379584512\nthp_fault_alloc 0\nthp_collapse_alloc 11741\n", + cg + "systemd-journald.service/io.stat": "9:0 \n8:16 rbytes=55209984 wbytes=2498768035840 rios=1317 wios=24719004 dbytes=0 dios=0\n" + + "8:0 rbytes=72953856 wbytes=2498768039936 rios=2194 wios=24719005 dbytes=0 dios=0\n" + + "9:2 rbytes=128163840 wbytes=2498724843520 rios=3511 wios=24707873 dbytes=0 dios=0\n", + cg + "sshd.service/memory.stat": "anon 1000\nfile_thp 0\n", + } + return tree(t, files) +} + +func TestNovoxBeforeTheChange(t *testing.T) { + a, err := novox(t).HugePages(10) + if err != nil { + t.Fatal(err) + } + if a.Mode != "always" || a.Defrag != "madvise" || a.Shmem != "advise" { + t.Fatalf("modes: %+v", a) + } + if len(a.PerSize) != 3 || a.PerSize[0].Size != "64kB" || a.PerSize[2].Size != "2048kB" || + a.PerSize[2].Enabled != "inherit" || a.PerSize[2].Means != "always" || a.PerSize[0].Means != "never" { + t.Fatalf("per size: %+v", a.PerSize) + } + if a.Meminfo["FileHugePages"] != 1654 || a.Meminfo["FilePmdMapped"] != 862 { + t.Fatalf("meminfo: %v", a.Meminfo) + } + if len(a.Units) != 2 || a.Units[0].Cgroup != "system.slice/systemd-journald.service" || + a.Units[1].Cgroup != "system.slice/fail2ban.service" { + t.Fatalf("units: %+v", a.Units) + } + // The mirror and its two members each count the write: the most any one saw, never their sum. + if a.Units[1].Written != 67723415552 || a.Units[1].FileTHP != 197132288 || a.Units[1].CollapsedByKhug != 550 { + t.Fatalf("fail2ban: %+v", a.Units[1]) + } + said := strings.Join(a.Said, "\n") + for _, want := range []string{"transparent huge pages: always, defrag madvise", "sizes not never: 2048kB always", + "file 1654 MiB (mapped at 2 MiB 862 MiB)", "system.slice/fail2ban.service: file in huge folios 188.0 MiB"} { + if !strings.Contains(said, want) { + t.Errorf("said lacks %q:\n%s", want, said) + } + } +} + +func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) { + a, err := novox(t).HugePages(1) + if err != nil || len(a.Units) != 1 { + t.Fatalf("%v %+v", err, a) + } + b, err := tree(t, map[string]string{"sys/kernel/mm/transparent_hugepage/enabled": "always [madvise] never\n"}).HugePages(10) + if err != nil || b.Mode != "madvise" || len(b.Units) != 0 || !strings.Contains(strings.Join(b.Said, "\n"), "no unit holds") { + t.Fatalf("%v %+v", err, b) + } +} + +func TestAKernelWithoutHugePagesIsAnError(t *testing.T) { + if _, err := tree(t, map[string]string{"proc/meminfo": ""}).HugePages(10); err == nil || + !strings.Contains(err.Error(), "built without transparent huge pages") { + t.Fatalf("%v", err) + } +} + +func TestTheArgumentIsBounded(t *testing.T) { + if _, err := whole(map[string]any{"units": -1.0}, "units", 10, 0, 100); err == nil { + t.Error("a negative count was accepted") + } + if n, _ := whole(map[string]any{"units": 1000.0}, "units", 10, 0, 100); n != 100 { + t.Errorf("not bounded: %d", n) + } +} diff --git a/modules/kernel/cmd/kernel-tools/machine.go b/modules/kernel/cmd/kernel-tools/machine.go new file mode 100644 index 00000000..b5f6c730 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/machine.go @@ -0,0 +1,65 @@ +package main + +// The machine as the tools see it: the files they read, under a root that tests replace with a stand-in +// tree. Everything read here is readable by the runtime's account; nothing is escalated. + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// Machine reads this machine's kernel files under Root ("/" on the machine). +type Machine struct { + Root string +} + +// ThisMachine is the machine the runtime launched this bundle on. +func ThisMachine() *Machine { return &Machine{Root: "/"} } + +func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) } + +func (m *Machine) read(p string) (string, error) { + b, err := os.ReadFile(m.path(p)) + return string(b), err +} + +func (m *Machine) dirs(p string) ([]string, error) { + entries, err := os.ReadDir(m.path(p)) + if err != nil { + return nil, err + } + var out []string + for _, e := range entries { + if e.IsDir() || e.Type()&fs.ModeSymlink != 0 { + out = append(out, e.Name()) + } + } + return out, nil +} + +// whole is a whole-number argument with a default, kept within bounds. +func whole(args map[string]any, key string, def, least, most int) (int, error) { + raw, present := args[key] + if !present || raw == nil { + return def, nil + } + f, ok := raw.(float64) + if !ok || f != float64(int(f)) { + return 0, fmt.Errorf("%s must be a whole number", key) + } + n := int(f) + if n < least { + return 0, fmt.Errorf("%s must be at least %d", key, least) + } + if n > most { + n = most + } + return n, nil +} + +// schema is a tool's input: its properties. +func schema(properties map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": properties} +} diff --git a/modules/kernel/cmd/kernel-tools/main.go b/modules/kernel/cmd/kernel-tools/main.go new file mode 100644 index 00000000..ec15c916 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/main.go @@ -0,0 +1,50 @@ +// The kernel module's tools bundle (novox/hq to-be 42 Phase 1 row 11, issue 746). +// +// Served by the node's runtime over MCP on stdio through the Go SDK (ADR 0188, ADR 0193). It only reads: +// the transparent huge pages the kernel runs with, and which units hold file pages in huge folios, which +// is what a small write to a mapped file costs a whole huge page of writeback for (issue 746). +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// binaryName is what the build names this bundle's executable: the manifest's `binary`. +const binaryName = "kernel-tools" + +func main() { + // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): kernel. + if err := stdio.Serve("", tools(ThisMachine())); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func tools(m *Machine) []stdio.Tool { + return []stdio.Tool{ + { + Name: "kernel_huge_pages", + Description: "Transparent huge pages on this machine: the mode in force (always, madvise or never) and defrag, " + + "each per-size control with what inherit resolves to, the machine's huge pages by kind (anonymous, " + + "shared memory, file, file mapped at 2 MiB) from /proc/meminfo, and the units holding the most file " + + "pages in huge folios (memory.stat file_thp) with their dirty and writeback bytes and the bytes their " + + "cgroup has written (io.stat, the most any one device saw). A unit with file_thp that writes far more " + + "than its files grow is paying a whole huge page of writeback per small write (novox/hq issue 746). " + + "Replaces cat /sys/kernel/mm/transparent_hugepage/enabled, grep Huge /proc/meminfo and reading " + + "memory.stat by hand. (r)", + Input: schema(map[string]any{ + "units": map[string]any{"type": "integer", "description": "how many units to name, the most file_thp first (default 10, 0 for none, at most 100)"}, + }), + Run: func(args map[string]any) (any, error) { + n, err := whole(args, "units", 10, 0, 100) + if err != nil { + return nil, err + } + return m.HugePages(n) + }, + }, + } +} diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go new file mode 100644 index 00000000..bcdc42e3 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -0,0 +1,239 @@ +package main + +// The module's shape (novox/hq issue 746): the setting's default is what the kernel boots with, so +// assigning the module changes nothing until a machine sets another value; the script is written into the +// module's own state and run by a sandboxed oneshot unit that is reloaded (applied again) when the script, +// and so the setting, changes and restarted only when the unit does; and the script itself sets, verifies, +// gives back and refuses as it says. + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { + m := manifest(t) + s := m.Settings["transparent-huge-pages"] + if s["kind"] != "preference" || s["default"] != "always" || s["why"] == "" { + t.Fatalf("the setting: %v", s) + } + var ids []string + for _, r := range m.Resources { + ids = append(ids, r["id"].(string)) + } + if strings.Join(ids, " ") != "state huge-pages unit apply" { + t.Fatalf("order: %v", ids) + } + script := m.resource(t, "huge-pages") + // The script runs as root, so the setting it is written from is set at the terminal alone (issue 339). + if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true || + !strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") { + t.Fatalf("the script: %v", script["path"]) + } + unit := m.resource(t, "unit")["content"].(string) + for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply", + "ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", + "DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes", + "NoNewPrivileges=yes"} { + if !strings.Contains(unit, line+"\n") { + t.Errorf("the unit lacks %q", line) + } + } + apply := m.resource(t, "apply") + if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" { + t.Fatalf("apply: %v", apply) + } + // A new value reloads (apply again); only a new unit restarts, whose stop would pass back through the + // mode the machine was found with. + if on := apply["restart-on"].([]any); len(on) != 1 || on[0] != "unit" { + t.Fatalf("restart-on: %v", on) + } + if on := apply["reload-on"].([]any); len(on) != 1 || on[0] != "huge-pages" { + t.Fatalf("reload-on: %v", on) + } + if h := apply["health"].(map[string]any); h["kind"] != "unit" { + t.Fatalf("health: %v", h) + } +} + +// bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage +// directory, the module's state, and a PATH whose journalctl records that it was asked to rotate. +type bed struct { + script, sys, state, rotated, path, bin string +} + +func newBed(t *testing.T, want, found string) bed { + return newBedIn(t, t.TempDir(), want, found) +} + +// newBedIn writes the script for want into dir; a second bed in the same dir shares the machine and the +// state, as a reload with a new value does. +func newBedIn(t *testing.T, dir, want, found string) bed { + t.Helper() + b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"), + rotated: filepath.Join(dir, "rotated")} + content := manifest(t).resource(t, "huge-pages")["content"].(string) + content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want) + content = strings.ReplaceAll(content, "${dir:state}", b.state) + if strings.Contains(content, "${") { + t.Fatalf("a substitution the test does not make is left in the script") + } + bin := filepath.Join(dir, "bin") + for _, d := range []string{b.sys, b.state, bin} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write(t, b.script, content, 0o755) + if found != "" { + write(t, filepath.Join(b.sys, "enabled"), found, 0o644) + } + write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755) + b.bin = bin + b.booted(t, "running") + b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH") + return b +} + +// booted makes the service manager say state for is-system-running. +func (b bed) booted(t *testing.T, state string) { + t.Helper() + write(t, filepath.Join(b.bin, "systemctl"), "#!/bin/sh\n[ \"$1\" = is-system-running ] && echo "+state+"\n", 0o755) +} + +func write(t *testing.T, p, content string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } +} + +func (b bed) run(t *testing.T, verb string) (string, error) { + t.Helper() + cmd := exec.Command("sh", b.script, verb, b.sys, b.state) + cmd.Env = append(os.Environ(), "PATH="+b.path) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func (b bed) file(t *testing.T, p string) string { + t.Helper() + raw, err := os.ReadFile(p) + if err != nil { + return "" + } + return string(raw) +} + +func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) { + // novox's file on 2026-10-14, read through node-reads. + b := newBed(t, "madvise", "[always] madvise never\n") + out, err := b.run(t, "apply") + if err != nil || !strings.Contains(out, "always -> madvise") { + t.Fatalf("apply: %v %s", err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" { + t.Fatalf("written: %q", got) + } + if got := b.file(t, filepath.Join(b.state, "found")); got != "always\n" { + t.Fatalf("found: %q", got) + } + if got := b.file(t, b.rotated); got != "--rotate\n" { + t.Fatalf("rotation: %q", got) + } + // Applied again (a restart with the same value): nothing written, nothing rotated, found kept. + if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise, as set") { + t.Fatalf("again: %v %s", err, out) + } + if got := b.file(t, b.rotated); got != "--rotate\n" { + t.Fatalf("rotated again: %q", got) + } + if out, err := b.run(t, "restore"); err != nil || !strings.Contains(out, "back to always") { + t.Fatalf("restore: %v %s", err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" { + t.Fatalf("restored: %q", got) + } + // Nothing of the module is kept once it is stopped. + if _, err := os.Stat(filepath.Join(b.state, "found")); !os.IsNotExist(err) { + t.Fatalf("found kept after restore: %v", err) + } +} + +func TestAReloadWithANewValueGoesStraightThereAndNeverRewritesFound(t *testing.T) { + dir := t.TempDir() + b := newBedIn(t, dir, "madvise", "[always] madvise never\n") + if out, err := b.run(t, "apply"); err != nil { + t.Fatalf("apply: %v %s", err, out) + } + // The setting changes to never: the mesh rewrites the script and reloads the unit, which applies again. + n := newBedIn(t, dir, "never", "") + out, err := n.run(t, "apply") + if err != nil || !strings.Contains(out, "madvise -> never") { + t.Fatalf("reload: %v %s", err, out) + } + if got := n.file(t, filepath.Join(n.state, "found")); got != "always\n" { + t.Fatalf("found rewritten by a reload: %q", got) + } +} + +func TestNoRotationWhileTheMachineIsStarting(t *testing.T) { + for _, state := range []string{"initializing", "starting"} { + b := newBed(t, "madvise", "[always] madvise never\n") + b.booted(t, state) + out, err := b.run(t, "apply") + if err != nil || !strings.Contains(out, "still starting") { + t.Fatalf("%s: %v %s", state, err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" { + t.Fatalf("%s: not set: %q", state, got) + } + if got := b.file(t, b.rotated); got != "" { + t.Fatalf("%s: rotated at boot: %q", state, got) + } + } +} + +func TestAlwaysIsSetWithoutRotating(t *testing.T) { + b := newBed(t, "always", "always [madvise] never\n") + if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise -> always") { + t.Fatalf("apply: %v %s", err, out) + } + if got := b.file(t, b.rotated); got != "" { + t.Fatalf("rotated for always: %q", got) + } +} + +func TestAValueTheKernelDoesNotTakeIsRefusedBeforeAnythingIsWritten(t *testing.T) { + b := newBed(t, "sometimes", "[always] madvise never\n") + out, err := b.run(t, "apply") + if err == nil || !strings.Contains(out, "takes always, madvise or never") { + t.Fatalf("accepted: %v %s", err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "[always] madvise never\n" { + t.Fatalf("written anyway: %q", got) + } +} + +func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root writes a read-only file") + } + b := newBed(t, "madvise", "[always] madvise never\n") + if err := os.Chmod(filepath.Join(b.sys, "enabled"), 0o444); err != nil { + t.Fatal(err) + } + if out, err := b.run(t, "apply"); err == nil { + t.Fatalf("a refused write passed: %s", out) + } +} + +func TestAKernelWithoutHugePagesIsSaid(t *testing.T) { + b := newBed(t, "madvise", "") + if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") { + t.Fatalf("%v %s", err, out) + } +} diff --git a/modules/kernel/cmd/kernel-tools/shape_test.go b/modules/kernel/cmd/kernel-tools/shape_test.go new file mode 100644 index 00000000..c30224f3 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/shape_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "encoding/json" + "os" + "testing" +) + +type resource map[string]any + +type manifestShape struct { + Module string `json:"module"` + Capabilities []string `json:"capabilities"` + Settings map[string]map[string]any `json:"settings"` + Tools []string `json:"tools"` + Resources []resource `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` +} + +func manifest(t *testing.T) manifestShape { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m manifestShape + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func (m manifestShape) resource(t *testing.T, id string) resource { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + t.Fatalf("no resource %s", id) + return nil +} + +// TestToolsAreTheManifests holds the served tools and the manifest's list to one another, and the +// bundle to the shape the builder compiles and the runtime loads. +func TestToolsAreTheManifests(t *testing.T) { + m := manifest(t) + names := map[string]bool{} + for _, tool := range tools(&Machine{Root: t.TempDir()}) { + if names[tool.Name] { + t.Errorf("%s is served twice", tool.Name) + } + names[tool.Name] = true + } + for _, want := range m.Tools { + if !names[want] { + t.Errorf("the manifest lists %s and the bundle does not serve it", want) + } + delete(names, want) + } + if len(names) != 0 { + t.Errorf("served and not listed: %v", names) + } + var tools map[string]any + for _, a := range m.Build.Artifacts { + if a["name"] == "tools" { + tools = a + } + } + if tools == nil || tools["kind"] != "bundle" || tools["language"] != "go" || tools["system"] != "arch" || + tools["from"] != "cmd/"+binaryName || tools["binary"] != binaryName { + t.Fatalf("the tools artifact: %v", tools) + } + if loads, _ := tools["loads"].([]any); len(loads) != 1 || loads[0] != binaryName { + t.Fatalf("loads: %v", tools["loads"]) + } +} diff --git a/modules/kernel/go.mod b/modules/kernel/go.mod new file mode 100644 index 00000000..748c098c --- /dev/null +++ b/modules/kernel/go.mod @@ -0,0 +1,5 @@ +module kernel + +go 1.22 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/kernel/go.sum b/modules/kernel/go.sum new file mode 100644 index 00000000..b474419e --- /dev/null +++ b/modules/kernel/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/kernel/module.json b/modules/kernel/module.json new file mode 100644 index 00000000..15077979 --- /dev/null +++ b/modules/kernel/module.json @@ -0,0 +1,71 @@ +{ + "module": "kernel", + "version": "1", + "capabilities": [ + "service-manager" + ], + "settings": { + "transparent-huge-pages": { + "kind": "preference", + "default": "always", + "why": "always is what the distribution's kernel boots with, so assigning the module changes nothing until a machine sets another value; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)" + } + }, + "tools": [ + "kernel_huge_pages" + ], + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": "." + }, + { + "id": "huge-pages", + "type": "file", + "path": "${dir:state}/huge-pages", + "mode": "0755", + "trusted": true, + "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" + }, + { + "id": "unit", + "type": "file", + "path": "/etc/systemd/system/mesh-kernel-huge-pages.service", + "mode": "0644", + "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n" + }, + { + "id": "apply", + "type": "service", + "unit": "mesh-kernel-huge-pages.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "unit" + ], + "reload-on": [ + "huge-pages" + ], + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/kernel-tools", + "binary": "kernel-tools", + "loads": [ + "kernel-tools" + ] + } + ] + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/answers.go b/modules/mesh-delivery/cmd/mesh-delivery/answers.go index 3323a5c5..d62685cd 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/answers.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/answers.go @@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool { return false } -// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. +// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose +// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so +// past the grace without its word it is listed by `stalled`, saying why it was not given. func (h *Holder) heldWalk(w Walk) bool { for _, d := range h.deliveries { if d.State.Final() { @@ -292,12 +294,28 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine { continue } w := s.walk + if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 { + // Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why. + says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+ + "merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8)) + if why := h.lapsedSaid(w); why != "" { + says += ": " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: its word is given at its turn — the operator's past the bound", Says: says}) + continue + } out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + out[len(out)-1].Says += h.endedSaid(w) + if hold := h.coverHeld(id); hold != "" { + out[len(out)-1].Says += "; it is held back meanwhile: " + hold + } } return out } @@ -344,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) { } case !w.Open() && now.Sub(s.at) > keptWalkFor: delete(h.walks, id) + delete(h.deliverErr, id) } } } @@ -356,6 +375,168 @@ type turnWant struct { whys []string } +// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live +// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none. +func (h *Holder) endedSaid(w Walk) string { + var ended []string + for _, d := range h.deliveries { + if !d.State.Final() { + continue + } + _, answers := walkAnswers(w, d) + if answers || d.Walk != nil && d.Walk.ID == w.ID { + ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + } + if len(ended) == 0 { + return "" + } + sort.Strings(ended) + return "; every delivery it carries ended: " + strings.Join(ended, ", ") +} + +// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that +// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for +// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it, +// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so +// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for +// itself, or by a person, gets nothing from it. Under the lock. +// +// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached +// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in +// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives +// nothing; lapsedBehind names that one. +func (h *Holder) lapsedOn(w Walk) []string { + out, _ := h.lapsedOf(w) + return out +} + +// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that +// lapsed whose merge the walk answers. Under the lock. +func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) { + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits { + return nil, nil + } + var out []string + behind := map[string]string{} + for _, d := range h.deliveries { + if !h.lapsedAnswered(w, d) { + continue + } + if before := h.lapsedBehind(w, d); before != "" { + behind[d.ID] = before + continue + } + out = append(out, d.ID) + } + sort.Strings(out) + return out, behind +} + +// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out. +func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool { + if d.MergedAs == "" || !h.stopLapsed(d) { + return false + } + if _, ok := walkAnswers(w, d); !ok { + return false + } + _, left := w.leftOut(d.Repository, d.MergedAs) + return !left +} + +// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this +// walk does not deliver with it, with its state; empty for none. +func (h *Holder) lapsedBehind(w Walk, d *Delivery) string { + g := h.groups[d.Group] + if d.Group == "" || g == nil { + return "" + } + members := h.membersOf(g) + byID := map[string]*Delivery{} + for _, m := range members { + byID[m.ID] = m + } + for _, m := range h.inOrder(g, members) { + if m.ID == d.ID { + return "" + } + o := byID[m.ID] + if o == nil || memberReached(o) || h.lapsedAnswered(w, o) { + continue + } + return fmt.Sprintf("%s (%s)", o.ID, o.State) + } + return "" +} + +// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet +// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock. +func (h *Holder) lapsedSaid(w Walk) string { + var parts []string + _, behind := h.lapsedOf(w) + ids := make([]string, 0, len(behind)) + for id := range behind { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id])) + } + if why := h.waitsWhy[w.ID]; why != "" { + parts = append(parts, "not let go yet: "+why) + } + if err := h.deliverErr[w.ID]; err != "" { + parts = append(parts, "the controller refused its word: "+err) + } + lapsed, _ := h.lapsedOf(w) + for _, id := range lapsed { + if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) { + parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it", + id, d.LetGo.Local().Format("15:04:05"))) + } + } + return strings.Join(parts, "; ") +} + +// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered. +func (h *Holder) lapsedOwed(w Walk) bool { + lapsed, behind := h.lapsedOf(w) + return len(lapsed) > 0 || len(behind) > 0 +} + +// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop +// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once, +// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words). +// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock. +func (h *Holder) lapsedTurns() { + for id, s := range h.walks { + for _, did := range h.lapsedOn(s.walk) { + d := h.deliveries[did] + if strings.Contains(d.LetGoWhy, lapsedWord(id)) { + continue + } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[id] + if want == nil { + want = &turnWant{} + h.turning[id] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+ + "was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+ + "walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository)) + } + } + } +} + +// lapsedWord is how the word a lapsed stop gives a walk names that walk. +func lapsedWord(walk string) string { return "the walk " + walk + " answers" } + // words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per // walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). // Under the lock. @@ -388,6 +569,9 @@ func (h *Holder) words() []func() { if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 { h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " + "asked to leave that merge out first" + if hold := h.coverHeld(walk); hold != "" { + h.waitsWhy[walk] = hold + } h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk, strings.Join(stopped, ", ")) continue @@ -403,9 +587,14 @@ func (h *Holder) words() []func() { defer h.mu.Unlock() delete(h.asking, walk) if err != nil { + if h.deliverErr == nil { + h.deliverErr = map[string]string{} + } + h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753) h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) return } + delete(h.deliverErr, walk) for _, id := range ids { d := h.deliveries[id] if d == nil { @@ -479,9 +668,28 @@ func (h *Holder) WalksWaiting() []StalledLine { switch { case len(held) > 0: out = append(out, held...) + case h.coverHeld(id) != "": + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound", + Says: "held back: " + h.coverHeld(id)}) case h.waitsWhy[id] != "": out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) + case !live && h.lapsedOwed(w): + // Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never + // unheld, while the controller has not taken it, with what holds it. + lapsed, _ := h.lapsedOf(w) + says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " + + "delivered since (novox/hq issue 753)" + if len(lapsed) > 0 { + says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " + + "whose merge it answers (novox/hq issue 753)" + } + if why := h.lapsedSaid(w); why != "" { + says += "; " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: says}) case !live: says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " + "`mesh-delivery.show` for the deliveries of " + w.Repository @@ -492,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine { "), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word" } out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited, - Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says}) + Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)}) } } sort.Slice(out, func(i, j int) bool { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/effects.go b/modules/mesh-delivery/cmd/mesh-delivery/effects.go index e7db65f2..b7a2115f 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/effects.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/effects.go @@ -150,6 +150,9 @@ func statusOf(d *Delivery) (string, string) { case Delivered: return "success", "delivered" case Failed: + if d.coveredStop() { + return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747) + } return "failure", clip("failed: "+why, 140) case Superseded: return "warning", clip("superseded: "+why, 140) @@ -178,7 +181,7 @@ func ViewBody(d *Delivery, g *Group) string { var b strings.Builder fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339)) if d.coveredStop() { - fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy) + fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid()) } if d.HeldWhy != "" && d.State == Held { fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index d6572f79..ded86f62 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -106,6 +106,8 @@ type fakeController struct { moves map[string][]string plans int down bool + // refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753). + refuseDeliver bool // onCheck runs as a check is asked, before it is answered: what the forge held at that moment. onCheck func() // planned is what Plan answers instead of "builds app", when set. @@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error { if c.down { return errors.New("down") } + if c.refuseDeliver { + return errors.New("the controller refuses the word") + } w, known := c.walks[walk] if !known || w.Delivery == nil || w.Delivery.Go != nil { return fmt.Errorf("%s waits for nobody", walk) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index da430dc2..a256007a 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -60,6 +60,8 @@ type Holder struct { waitsWhy map[string]string // asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile. asking map[string]bool + // deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753). + deliverErr map[string]string // leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299). leaving map[string][]string // leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped). @@ -67,6 +69,13 @@ type Holder struct { // leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and // to stop the walk whole. leaveRefused map[string]string + // coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the + // forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of + // them is held back meanwhile, never asked to leave it out before the answer. + coverAsked map[string]bool + // coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past + // coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747). + coverAskedSince map[string]time.Time // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). tried owedTries // marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go). @@ -1112,6 +1121,48 @@ func (h *Holder) plan() []func() { // A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what // that walk did not build, a later delivered merge may have; and when the cover test says none did, past the // grace, it is stopped saying so — never left waiting for a start that cannot come. + // A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its + // change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it + // (novox/hq issue 747). Read each tick, from what the walks and the kept deliveries say, so one stuck before this + // was built clears by itself after a restart. + h.coverAsked = map[string]bool{} + if h.coverAskedSince == nil { + h.coverAskedSince = map[string]time.Time{} + } + pending := map[string]bool{} + for _, d := range h.deliveries { + if d.State != Failed && d.State != Stopped || d.coveredStop() { + continue + } + by, ask := h.walkCover(d) + if len(ask) > 0 && by == "" { + pending[d.ID] = true + since, asked := h.coverAskedSince[d.ID] + if !asked { + since = now + h.coverAskedSince[d.ID] = now + } + // Held back only within the bound: a forge that keeps erroring never holds a walk for good. + h.coverAsked[d.ID] = now.Sub(since) < coverAskBound + } + if by != "" { + d.DeliveredBy = by + h.tellCovered(d, now) + continue + } + for _, in := range ask { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + + for id := range h.coverAskedSince { + if !pending[id] { + delete(h.coverAskedSince, id) + } + } + for _, d := range h.deliveries { ended := d.endedUnstarted() if d.coveredStop() && !d.CoverTold { @@ -1195,6 +1246,8 @@ func (h *Holder) plan() []func() { } h.turn(d, "its turn: a delivery in no group") } + // A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753). + h.lapsedTurns() // The word, once per walk, when every live delivery it carries is at its turn. asks = append(asks, h.words()...) @@ -1273,7 +1326,7 @@ func (h *Holder) laterOnItsWay(d *Delivery) string { // stays final, and the bus's transition event is not said again. func (h *Holder) tellCovered(d *Delivery, now time.Time) { d.CoverTold = true - line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid())) + line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid())) d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now}) if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) { d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now}) @@ -1283,7 +1336,7 @@ func (h *Holder) tellCovered(d *Delivery, now time.Time) { state, desc := statusOf(d) d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now}) } - h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid()) + h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid()) h.keep(d) } @@ -1294,6 +1347,104 @@ func strandedUncovered(d *Delivery) bool { return d.State == Stopped && d.Stranded != "" && d.CoveredBy == "" } +// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or +// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it +// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or +// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not +// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every +// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own +// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its +// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records +// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart. +func (h *Holder) walkCover(d *Delivery) (string, []string) { + if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 { + return "", nil + } + var ask []string + for _, c := range h.doneWalksOf(d.Repository) { + w := c.walk + if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) { + continue + } + var missing []string + for _, m := range d.Plan.Moved { + unchanged := false + if d.Walk != nil { + _, unchanged = d.Walk.Unchanged[m] + } + if !containsString(w.Moved, m) && !unchanged { + missing = append(missing, m) + } + } + if len(missing) > 0 { + continue + } + if !sameCommit(c.commit, d.MergedAs) { + yes, known := d.ContainedIn[c.commit] + if !known && !containsString(ask, c.commit) { + ask = append(ask, c.commit) + } + if !yes { + continue + } + } + return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8), + orNone(d.Base), shortOf(d.MergedAs, 8)), nil + } + return "", ask +} + +// doneWalk is a finished walk and the commit of its repository it carried. +type doneWalk struct { + walk *WalkSeen + commit string +} + +// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it +// holds, and each delivery's kept walk with the commit of its repository that walk carried for it. +func (h *Holder) doneWalksOf(repository string) []doneWalk { + byID := map[string]doneWalk{} + for id, s := range h.walks { + w := s.walk + if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) { + continue + } + byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit} + } + for _, o := range h.deliveries { + if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) { + continue + } + if _, known := byID[o.Walk.ID]; known { + continue + } + commit := o.AnsweredBy + if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) { + commit = o.MergedAs + } + if commit == "" { + continue + } + byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit} + } + out := make([]doneWalk, 0, len(byID)) + for _, c := range byID { + out = append(out, c) + } + sort.Slice(out, func(i, j int) bool { + if !out[i].walk.Created.Equal(out[j].walk.Created) { + return out[i].walk.Created.Before(out[j].walk.Created) + } + return out[i].walk.ID < out[j].walk.ID + }) + return out +} + +// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed). +func (h *Holder) stopLapsed(d *Delivery) bool { + return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] }) +} + // unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read // again after it: what holds it for a person, unless something already ends it. func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool { @@ -1736,7 +1887,8 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa // A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a // covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq // issues 442 and 445). - if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" { + // A stop or failure a later walk delivered since counts as reached too (novox/hq issues 480 and 747). + if memberReached(d) { continue } if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) { @@ -1761,7 +1913,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa } if w.why != "" { for _, later := range ordered[i+1:] { - if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, + if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, d.ID, w.why)); a != nil { asks = append(asks, a) } @@ -1788,9 +1940,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa continue } return asks + case h.stopLapsed(d): + // Stopped for a member whose change reached the machines since: it stops nothing more. Its own change is + // delivered by a later walk of its repository that contains it, which then covers it (novox/hq issue 747). + continue case d.State == Failed || d.State == Stopped: for _, later := range ordered[i+1:] { - if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { + if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { asks = append(asks, a) } } @@ -2028,7 +2184,7 @@ func mayCarry(c, d *Delivery, module string) bool { // controller ends the walk and walks the other merges again without it, and without each later merge of its // repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked // once, by the first member's ask, after the pass: a member is never walked again because its stop came second. -func (h *Holder) stopInGroup(d *Delivery, why string) func() { +func (h *Holder) stopInGroup(d *Delivery, cause, why string) func() { if d == nil || d.State.Final() { return nil } @@ -2037,6 +2193,7 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() { h.refuse(err) return nil } + d.StoppedFor = cause // the member it was stopped for: once that one is delivered, the stop lapses (novox/hq issue 747) h.moved(d, []Transition{t}) // Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review // of #221): a merge walked again and then stopped by its group is left out there too. @@ -2118,6 +2275,12 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) { if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" { continue } + // A stop whose change a later walk delivered since, or one its group made for a member delivered since, + // holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq + // issue 747). + if d.coveredStop() || h.stopLapsed(d) { + continue + } // By the merges it names alone: its own commit may be another's merge it no longer answers. if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool { return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) @@ -2135,6 +2298,51 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) { return out, why } +// coverAwaited is whether a stopped merge a walk carries, given as `@`, waits for the forge's +// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747). +// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first +// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock. +func (h *Holder) coverAwaited(merges []string) (time.Time, bool) { + var since time.Time + awaited := false + for _, d := range h.deliveries { + if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) { + continue + } + for _, id := range []string{d.ID, d.stoppedFor()} { + if id == "" || !h.coverAsked[id] { + continue + } + if at := h.coverAskedSince[id]; !awaited || at.Before(since) { + since = at + } + awaited = true + } + } + return since, awaited +} + +// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped +// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so +// a forge that cannot answer is loud. Under the lock. +func (h *Holder) coverHeld(walk string) string { + without, _ := h.carriesStopped(walk) + if len(without) == 0 { + return "" + } + since, awaited := h.coverAwaited(without) + if !awaited { + return "" + } + return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+ + "finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)", + strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05")) +} + +// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later +// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out. +const coverAskBound = 10 * time.Minute + // lastWhyOf is the reason of a delivery's last transition. func lastWhyOf(d *Delivery) string { if len(d.Transitions) == 0 { @@ -2170,6 +2378,9 @@ func (h *Holder) leaveStopped() []func() { if without, _ := h.carriesStopped(id); len(without) > 0 { said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+ "leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", ")) + if hold := h.coverHeld(id); hold != "" { + said = "its walk " + id + " is held back: " + hold + } if r := h.leaveRefused[id]; r != "" { said += "; the controller refused: " + r } @@ -2193,6 +2404,9 @@ func (h *Holder) leaveStopped() []func() { delete(h.leaveRefused, id) continue } + if _, awaited := h.coverAwaited(without); awaited { + continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747) + } // Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should // the ask have gone nowhere. if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute { @@ -2314,18 +2528,28 @@ func againWords(a StopAnswer) string { func GroupState(g *Group, members []*Delivery) (string, string) { count := map[State]int{} reached, ended := 0, 0 - var uncovered []string + var uncovered, lapsed []string + byID := map[string]*Delivery{} + for _, d := range members { + byID[d.ID] = d + } for _, d := range members { - count[d.State]++ if d.State.Final() { ended++ } switch { - case d.State.Reached(), d.State == Superseded && d.CoveredBy != "": + case memberReached(d): + // Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the + // group (novox/hq issue 747). reached++ + continue + case stopLapsed(d, func(id string) *Delivery { return byID[id] }): + lapsed = append(lapsed, d.ID) + continue case d.State == Superseded: uncovered = append(uncovered, d.ID) } + count[d.State]++ } n := len(members) switch { @@ -2335,6 +2559,11 @@ func GroupState(g *Group, members []*Delivery) (string, string) { return "stopped", "a member was stopped" case n > 0 && reached == n: return "delivered", "every member is delivered" + case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n: + // Stopped for a member delivered since: its own change waits for a later walk of its repository, which then + // covers it (novox/hq issue 747). + return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+ + "later walk of its repository", reached, n, strings.Join(lapsed, ", ")) case g.Closed && n > 0 && ended == n: return "stopped", supersededWhy + strings.Join(uncovered, ", ") case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" || diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go new file mode 100644 index 00000000..7646ec14 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go @@ -0,0 +1,258 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336 +// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge, +// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still +// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge +// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of +// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is +// still undelivered. +// +// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the +// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge. + +const ( + g747 = "fix/a-module-moves-only-when-its-program-changes" + ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog" + head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452" + head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6" + head301, merge301 = "d94764f00168471f", "2edef5a00ae05605" + merge307 = "0f76b6e2aaaaaaaa" +) + +var ( + at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC) + id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301) + unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"} +) + +// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df. +func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) } + +// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged. +func night747With(t *testing.T, contained bool, built []string) *world { + t.Helper() + return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built) +} + +// night747Over is night747With with the forge set as given: one that is not told the pair cannot say. +func night747Over(t *testing.T, set func(*fakeForge), built []string) *world { + t.Helper() + at := at747 + failedAt := at.Add(11 * time.Minute) + store := newMemStore() + moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747 + if built != nil { + moved339, unchanged339 = built, map[string]string{} + for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} { + if !containsString(built, m) { + unchanged339[m] = "its build source holds none of the files #339 changed" + } + } + } + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, + Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true, + Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2, + Unchanged: unchangedRouteProxy747}, + Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed, + Why: "its walk failed, carrying its own commit"}}} + m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726", + State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute), + AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"), + // Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so. + Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true, + Moved: moved339, Unchanged: unchanged339, + Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2}, + Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed, + Why: "its walk failed, carrying its own commit"}}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), + Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} + for _, d := range []*Delivery{m336, m339, m301} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed", + OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + // The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word. + ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}}) + ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute) + forge := newFakeForge() + set(forge) + w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +func groupState747(w *world) (string, string) { + g := w.h.groups[g747] + return GroupState(g, w.h.membersOf(g)) +} + +// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the +// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged. +// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose +// own walk was resumed to done, is covered by it too. +func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) { + w := night747(t, true) + d := w.delivery(id336) + if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") || + !strings.Contains(desc, "plan-339") { + t.Fatalf("#336's status reads %s %q", state, desc) + } + if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") { + t.Fatalf("the forge was told %q of #336", s) + } + if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") { + t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc) + } + if state, why := groupState747(w); state == "failed" || state == "stopped" { + t.Fatalf("the group reads %s: %s", state, why) + } + // Told once, and kept: a restart tells nothing again. + notes := len(w.forge.notesOn(merge336)) + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute)) + again.settleAll() + if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" { + t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes) + } +} + +// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone, +// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the +// group waits for that walk to deliver the catalogue member's change. +func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) { + w := night747(t, true) + if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") { + t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 { + t.Fatalf("the catalogue's walk is held back for %v", stopped) + } + if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) { + t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why) + } + // Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is + // delivered. + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk("plan-cat") + sent := w.now + r.Modules = map[string]*WalkModule{} + for _, m := range []string{"build-agent", "route-proxy", "gitea"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") { + t.Fatalf("the catalogue member reads %s %q", state, desc) + } + if state, why := groupState747(w); state != "delivered" { + t.Fatalf("the group reads %s: %s", state, why) + } +} + +// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change +// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries +// the stopped member's merge is asked to leave it out. +func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) { + w := night747(t, false) + d := w.delivery(id336) + if state, desc := statusOf(d); d.State != Failed || state != "failure" { + t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc) + } + if state, _ := groupState747(w); state != "failed" { + t.Fatalf("the group reads %s", state) + } + if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) { + t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts) + } +} + +// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk +// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only +// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue. +func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) { + w := night747With(t, true, []string{"mesh-controller"}) + if state, desc := statusOf(w.delivery(id336)); state != "failure" { + t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc) + } + if state, _ := groupState747(w); state != "failed" { + t.Fatalf("the group reads %s", state) + } + if len(w.ctl.withouts) == 0 { + t.Fatal("the catalogue's walk was not asked to leave the stopped merge out") + } +} + +// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on +// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the +// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk +// is asked to leave the stopped merge out, as with no later walk at all. +func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) { + for _, c := range []struct { + name string + set func(*fakeForge) + }{ + {"erroring", func(f *fakeForge) { f.containsDown = true }}, + {"not answering for the pair", func(*fakeForge) {}}, + } { + t.Run(c.name, func(t *testing.T) { + w := night747Over(t, c.set, nil) + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + if state, desc := statusOf(w.delivery(id336)); state != "failure" { + t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc) + } + said := "" + for _, l := range w.h.WalksWaiting() { + if l.Walk == "plan-cat" { + said += l.Says + } + } + w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it + w.settleAll() + for _, l := range w.h.Stalled() { + if l.Walk == "plan-cat" { + said += " | " + l.Says + } + } + if strings.Count(said, "the forge is asked since") != 2 { + t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + w.later(coverAskBound + time.Minute) + w.settleAll() + if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) { + t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts) + } + }) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go new file mode 100644 index 00000000..8bb50c99 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go @@ -0,0 +1,292 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at +// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its +// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery +// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later +// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's +// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word +// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries. +// +// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart. + +const ( + merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307" + plan753 = "plan-1792012015539062661" +) + +var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307) + +// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304 +// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting. +func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) } + +// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301. +func night753With(t *testing.T, stop Transition, more ...*Delivery) *world { + t.Helper() + return night753Over(t, stop, nil, nil, more...) +} + +// night753Over is night753With with more merges named by the walk and the controller set as given before the first +// tick. +func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world { + t.Helper() + at := at747 + store := newMemStore() + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true, + Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}} + if stop.By == byOwner { + m301.StoppedFor = id336 + } + m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events", + State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304, + MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"), + // Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747). + DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)", + CoverTold: true, + Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main", + TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)}, + Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop, + Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}} + m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check", + State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307, + MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")} + for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + inGroup, order := []*Delivery{m301, m336}, []string{id336} + for _, d := range more { + if d.Group == g747 { + inGroup, order = append(inGroup, d), append(order, d.ID) + } + } + order = append(order, id301) + var ids []string + for _, d := range inGroup { + ids = append(ids, d.ID) + } + g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering", + OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}} + ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307}, + {Repository: cat747, Commit: merge307}}}}) + if len(merges) > 0 { + r := ctlr.walk(plan753) + r.Delivery.Merges = append(merges, r.Delivery.Merges...) + ctlr.put(r) + } + if set != nil { + set(ctlr) + } + ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt + w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since. +func groupStop753() Transition { + return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"} +} + +func saidOf(lines []StalledLine, walk string) (string, State) { + said, state := "", State("") + for _, l := range lines { + if l.Walk == walk { + said, state = said+l.Says, l.State + } + } + return said, state +} + +// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for +// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this +// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld. +func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) { + w := night753(t, groupStop753()) + if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 { + t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers) + } + d := w.delivery(id301) + if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") { + t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld { + t.Fatalf("the walk reads unheld: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + // Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747). + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk(plan753) + sent := w.now + for _, m := range []string{"build-agent", "gitea", "route-proxy"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) { + t.Fatalf("#301 reads %s %q", state, desc) + } +} + +// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among +// the walk's, never a way past a live delivery it carries. +func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) { + at := at747 + held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308, + Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour), + MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main", + Waits: true, Created: at.Add(46 * time.Minute)}} + w := night753With(t, groupStop753(), held) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held { + t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said) + } +} + +// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out, +// never let go for it. +func TestAPersonsStopGivesNoWord(t *testing.T) { + w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen", + Why: "stopped by hand"}) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers) + } + if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 { + t.Fatal("the walk carrying a person's stop was neither left without it nor stopped") + } +} + +// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it +// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never +// only that nothing holds it. +func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) { + w := night753(t, groupStop753()) + // The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed + // gives it its word, and no live one holds it. + r := w.ctl.walk(plan753) + r.Delivery.Go = nil + r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}} + w.h.WalkMoved(w.ctl.put(r)) + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state != Unheld || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307) + } + w.settleAll() + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ = saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled` reads %q", said) + } +} + +// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk. +func m299() *Delivery { + at := at747 + return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299, + Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute), + Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"), + Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} +} + +// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and +// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land +// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and +// is let go. +func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) { + w := night753With(t, groupStop753(), m299()) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers) + } + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") { + t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said) + } + w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil, + m299()) + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers) + } +} + +// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past +// the grace `stalled` lists the walk, with the refusal. +func TestARefusedWordIsListedByStalled(t *testing.T) { + w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true }) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, state := saidOf(w.h.Stalled(), plan753) + if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") { + t.Fatalf("`stalled` reads %s %q", state, said) + } + // Once the controller takes it, the walk is let go. + w.ctl.refuseDeliver = false + w.settleAll() + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers) + } +} + +// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person, +// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the +// grace `stalled` lists the walk, saying why. +func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) { + at := at747 + m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305, + Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), + MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready, + To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}} + w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}}, + func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ := saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") || + !strings.Contains(said, "whose delivery was stopped") { + t.Fatalf("`stalled` reads %q", said) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index ec3e2e1d..2d2a0c67 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -83,6 +83,14 @@ type Delivery struct { // CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note // (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too. CoverTold bool `json:"cover_told,omitempty"` + // DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose + // commit contains its merge and which built every module it moves — each it did not build, its own walk said + // unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered + // stop's does (issue 480), and it reads covered: its group counts it as delivered. + DeliveredBy string `json:"delivered_by,omitempty"` + // StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that + // member's change reached the machines, the stop has lapsed, and holds no walk of its repository back. + StoppedFor string `json:"stopped_for,omitempty"` // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. @@ -303,18 +311,77 @@ func (d *Delivery) FromForgesWord() bool { // shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose // pull request the forge may no longer hold — or holds under its number another one. -// coveredStop is whether a delivery stopped because its walk ended done before its word without building its modules -// (novox/hq issue 474) was covered since: a later merge into its trunk that contains it was delivered, carrying every -// module it moves to every machine running it (ADR 0270's cover test, kept as CoveredBy). Its state stays stopped — -// final, as a covered superseded member stays superseded (issue 442) — and it reads covered, by that merge (novox/hq -// issue 480). A stop for any other reason never reads covered. +// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop +// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a +// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running +// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk +// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state +// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or +// walk. A person's stop with a cover found only while it was held keeps its error. func (d *Delivery) coveredStop() bool { - return d.State == Stopped && d.Stranded != "" && d.CoveredBy != "" + switch d.State { + case Stopped: + return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != "" + case Failed: + return d.DeliveredBy != "" + } + return false } -// coveredStopSaid is how a covered stop reads: on the forge's status, in its view and note, and in `show`. +// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`. func (d *Delivery) coveredStopSaid() string { - return "covered after its stop: a later merge that contains it was delivered: " + d.CoveredBy + ended := "its stop" + if d.State == Failed { + ended = "its failure" + } + if d.Stranded != "" && d.CoveredBy != "" { + return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy + } + return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy +} + +// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to +// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442), +// or a stop or failure covered since (issues 480 and 747). +func memberReached(d *Delivery) bool { + return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop() +} + +// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the +// stop's reason for a stop made before ("its group stopped: …"). +func (d *Delivery) stoppedFor() string { + if d.StoppedFor != "" || d.State != Stopped || d.Group == "" { + return d.StoppedFor + } + prefix := "its group " + d.Group + " stopped: " + for i := len(d.Transitions) - 1; i >= 0; i-- { + t := d.Transitions[i] + if t.Event != EvStop { + continue + } + if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) { + return "" + } + member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ") + return strings.TrimRight(member, ",;:") + } + return "" +} + +// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's +// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own +// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find +// gives a delivery by its id. +func stopLapsed(d *Delivery, find func(string) *Delivery) bool { + if d.State != Stopped || d.coveredStop() { + return false + } + by := d.stoppedFor() + if by == "" || by == d.ID { + return false + } + o := find(by) + return o != nil && memberReached(o) } func (d *Delivery) shownOnItsPullRequest() bool { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index b95aacd2..b2dfed42 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -72,6 +72,10 @@ func (h *Holder) waitsFor(d *Delivery) string { return "its walk: " + clip(d.Walk.Note, 120) } case Stopped: + if h.stopLapsed(d) { + return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() + + ", whose change was delivered since (novox/hq issue 747)" + } if strandedUncovered(d) { return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " + "listed by stalled until one does (novox/hq issue 480)" diff --git a/modules/route-proxy/Dockerfile b/modules/route-proxy/Dockerfile index b556c2f6..7e7fe648 100644 --- a/modules/route-proxy/Dockerfile +++ b/modules/route-proxy/Dockerfile @@ -20,7 +20,7 @@ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /mesh-route-proxy ./examples/route-proxy +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -buildvcs=false -ldflags=-buildid= -o /mesh-route-proxy ./examples/route-proxy # A small runtime with the public CA roots the ACME client needs to reach a real authority, and run # as root so it can bind :80 and :443 — the two privileged ports a public front door listens on.