Merge pull request 'Give the forge bridge a merge commit's trees in one call, and each open pull request's head branch (design 47 R3c, issue 1234)' (#374) from feat/delivery-parts into main

modules/gitea/client.ts

@@ -3,6 +3,7 @@
 // gitea. Both this module's tools and its events entrypoint import it, and nothing outside gitea
 // does.
 
+import { createHash } from "node:crypto";
 import { readFileSync } from "node:fs";
 import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js";
 import { MAIN_BRANCH, mainRuleBody, protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
@@ -21,6 +22,19 @@ function commitSegment(sha: unknown): string {
   if (!isCommit(v)) throw new Refused(`${JSON.stringify(sha)} is not a commit. Nothing was sent`);
   return v;
 }
+/** git's hash of a tree from its entries: sha1 of "tree <length>\0" and, for each entry in git's order (byte order of the
+ *  name, a tree compared as its name and a slash), "<mode> <name>\0" and the 20 raw bytes of its sha. git writes a
+ *  tree's mode 40000 where the forge says 040000. */
+export function gitTreeHash(entries: { path: string; mode: string; type: string; sha: string }[]): string {
+  const ordered = entries
+    .map((e) => ({ e, key: Buffer.from(e.path + (e.type === "tree" ? "/" : "")) }))
+    .sort((a, b) => Buffer.compare(a.key, b.key));
+  const body = Buffer.concat(ordered.map(({ e }) => {
+    if (!/^[0-9a-f]{40}$/i.test(String(e.sha))) throw new Error(`the forge named no sha for ${e.path}`);
+    return Buffer.concat([Buffer.from(`${String(e.mode).replace(/^0+/, "")} ${e.path}\0`), Buffer.from(e.sha, "hex")]);
+  }));
+  return createHash("sha1").update(Buffer.concat([Buffer.from(`tree ${body.length}\0`), body])).digest("hex");
+}
 export type { BranchProtection, ProtectionWanted } from "./protection.js";
 
 /** A repository, trimmed to what the mesh cares about. */
@@ -92,6 +106,8 @@ export interface OpenPull {
   repo: string;
   number: number;
   base: string;
+  /** The pull request's head branch: what a delivery group is named by (novox/hq ADR 0249). */
+  head?: string;
   head_sha: string;
   html_url?: string;
   /** The contexts the base branch's protection requires; none when it requires none, or cannot be read. */
@@ -374,7 +390,7 @@ export class GiteaClient {
       const rule = rules.get(base);
       const checks = await this.headChecks(found.owner, found.name, pull.head_sha);
       const labels = found.labels ?? [];
-      out.push({ owner: found.owner, repo: found.name, number: pull.number, base: pull.base ?? "", head_sha: pull.head_sha,
+      out.push({ owner: found.owner, repo: found.name, number: pull.number, base: pull.base ?? "", head: pull.head, head_sha: pull.head_sha,
         html_url: pull.html_url, required: rule?.enable_status_check ? (rule.status_check_contexts ?? []) : [],
         statuses: Object.keys(checks), checks, labels: labels.map((l) => l.name) });
     }
@@ -510,6 +526,43 @@ export class GiteaClient {
     return out;
   }
 
+  /** **A commit's trees: its root, and the tree of every directory that holds a `module.json`** (novox/hq issue 1234,
+   *  ADR 0437 §4 and §5), from the forge's one recursive listing of the commit, read page by page. A directory is
+   *  spelled as the repository spells it, no slash at either end, the root as `""`.
+   *
+   *  **A listing the forge cut is no listing** (it says `truncated` on a page with nothing more to read, or the pages
+   *  run past a bound): the call fails naming the repository and the commit, never answering the directories it did
+   *  see, since a directory left out would be taken for one without a module. Reads only. */
+  async commitTrees(owner: string, repo: string, sha: string): Promise<{ root: string; trees: Record<string, string> }> {
+    const base = `${repoPath(owner, repo)}/git/trees/${commitSegment(sha)}?recursive=true`;
+    const cut = (why: string) => new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} was cut (${why}); no trees are answered from a part`);
+    const entries: any[] = [];
+    for (let page = 1; ; page++) {
+      if (page > 200) throw cut("more than 200 pages");
+      const got = await this.request<any>(`${base}&page=${page}&per_page=1000`);
+      if (!Array.isArray(got?.tree)) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} has no tree`);
+      entries.push(...got.tree);
+      if (got.truncated !== true) break;
+      // Truncated with nothing more to ask for: the forge itself could not give the rest.
+      if (got.tree.length === 0 || entries.length >= Number(got.total_count ?? 0)) throw cut("truncated");
+    }
+    // **The listing's top `sha` is the commit's, not the root tree's** (seen live, novox/mesh-delivery at 7f61817), so the
+    // root tree is hashed from the root's own entries, as git does (gitTreeHash).
+    const root = gitTreeHash(entries.filter((e) => typeof e?.path === "string" && !e.path.includes("/")));
+    const trees: Record<string, string> = {};
+    const dirOf = new Map<string, string>();
+    for (const e of entries) if (e?.type === "tree" && typeof e.path === "string") dirOf.set(e.path, String(e.sha));
+    for (const e of entries) {
+      if (e?.type !== "blob" || typeof e.path !== "string") continue;
+      if (e.path !== "module.json" && !e.path.endsWith("/module.json")) continue;
+      const dir = e.path === "module.json" ? "" : e.path.slice(0, -"/module.json".length);
+      const tree = dir === "" ? root : dirOf.get(dir);
+      if (!tree) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} holds ${e.path} but not the tree of ${dir}`);
+      trees[dir] = tree;
+    }
+    return { root, trees };
+  }
+
   /** Whether a repository holds a file at a commit: true, false for a 404, thrown when the forge cannot say. */
   async holdsFile(owner: string, repo: string, sha: string, file: string): Promise<boolean> {
     return this.exists(`${repoPath(owner, repo)}/contents/${encodePath(file)}?ref=${encodeURIComponent(sha)}`);

modules/gitea/guard.ts

@@ -94,6 +94,8 @@ export const QUERY_KEYS: ReadonlySet<string> = new Set([
   "limit", "page", "state", "type", "q", "sort", "order", "since", "before", "ref", "sha", "path", "labels",
   "milestones", "keyword", "stat", "verification", "files", "not", "mode", "include_desc", "archived", "private",
   "exclusive", "uid", "priority_repo_id", "created_by", "assigned_by", "mentioned_by", "whitespace", "binary",
+  // Read flags of the forge's tree listing (gitea_commit_trees, novox/hq issue 1234): they widen what one read answers, never what a call may write.
+  "recursive", "per_page",
 ]);
 
 /** The query a call sends: only allow-listed keys, each as given; refused when any other key is there. */

modules/gitea/test/commit-trees.test.ts

@@ -0,0 +1,173 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { createServer } from "node:http";
+
+// Fixtures follow Gitea's GitTreeResponse as its API documents it (swagger: sha, url, tree[{path, mode, type, size,
+// sha, url}], truncated, page, total_count) and as the forge answers GET /repos/{o}/{r}/git/trees/{sha}; the entry
+// fields are those, with sha values written out. The live capture (1234/AC6): GET
+// /repos/novox/mesh-delivery/git/trees/7f61817 (non-recursive) answered the COMMIT's sha at the top, not the root
+// tree's; the root tree is computed from the root's own entries, as git does.
+import { createHash } from "node:crypto";
+
+const COMMIT = "c".repeat(40);
+const h = (c: string) => c.repeat(40);
+const entry = (path: string, type: "tree" | "blob", sha: string) => ({
+  path, mode: type === "tree" ? "040000" : "100644", type, size: type === "tree" ? 0 : 12, sha, url: `u/${sha}`,
+});
+const LISTING = [
+  entry("README.md", "blob", h("1")),
+  entry("module.json", "blob", h("2")),
+  entry("modules", "tree", h("3")),
+  entry("modules/a", "tree", h("4")),
+  entry("modules/a/module.json", "blob", h("5")),
+  entry("modules/a/main.go", "blob", h("6")),
+  entry("modules/b", "tree", h("7")),
+  entry("modules/b/c", "tree", h("8")),
+  entry("modules/b/c/module.json", "blob", h("9")),
+  entry("modules/d", "tree", h("a")),
+  entry("modules/d/notes.md", "blob", h("b")),
+];
+
+/** git's own hash of a tree from its entries (an independent restatement for the fixtures): mode without leading zero. */
+function gitTree(es: { path: string; type: string; sha: string; mode: string }[]): string {
+  const ordered = es.map((e) => ({ ...e, key: e.path + (e.type === "tree" ? "/" : "") })).sort((a, b) => Buffer.compare(Buffer.from(a.key), Buffer.from(b.key)));
+  const body = Buffer.concat(ordered.map((e) => Buffer.concat([Buffer.from(`${e.mode.replace(/^0+/, "")} ${e.path}\0`), Buffer.from(e.sha, "hex")])));
+  return createHash("sha1").update(Buffer.concat([Buffer.from(`tree ${body.length}\0`), body])).digest("hex");
+}
+const ROOT = gitTree(LISTING.filter((e) => !e.path.includes("/")));
+
+async function serve(pages: any[]) {
+  const { GiteaClient } = await import("../client.ts");
+  const asked: string[] = [];
+  const server = createServer((req, res) => {
+    asked.push(req.url ?? "");
+    const url = new URL(req.url ?? "", "http://x");
+    if (url.pathname !== `/api/v1/repos/novox/hq/git/trees/${COMMIT}` || url.searchParams.get("recursive") !== "true") {
+      res.statusCode = 404;
+      return res.end("{}");
+    }
+    const page = Number(url.searchParams.get("page") ?? 1);
+    res.end(JSON.stringify(pages[page - 1] ?? { sha: COMMIT, tree: [], truncated: false, page, total_count: 0 }));
+  });
+  await new Promise<void>((r) => server.listen(0, r));
+  const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
+  return { client, server, asked };
+}
+
+test("a commit's trees: the root and the tree of each directory holding a module.json", async () => {
+  // 1234/AC1 and 1234/AC4
+  const { client, server } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
+  try {
+    const got = await client.commitTrees("novox", "hq", COMMIT);
+    assert.deepEqual(got, { root: ROOT, trees: { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") } });
+  } finally {
+    server.close();
+  }
+});
+
+test("a listing the forge pages is read page by page", async () => {
+  // 1234/AC1
+  const { client, server, asked } = await serve([
+    { sha: COMMIT, tree: LISTING.slice(0, 5), truncated: true, page: 1, total_count: LISTING.length },
+    { sha: COMMIT, tree: LISTING.slice(5, 9), truncated: true, page: 2, total_count: LISTING.length },
+    { sha: COMMIT, tree: LISTING.slice(9), truncated: false, page: 3, total_count: LISTING.length },
+  ]);
+  try {
+    const got = await client.commitTrees("novox", "hq", COMMIT);
+    assert.deepEqual(got.trees, { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") });
+    assert.equal(asked.length, 3, `asked ${asked.join(" ")}`);
+  } finally {
+    server.close();
+  }
+});
+
+test("a listing the forge says is cut fails, naming the repository and the commit, and answers no trees", async () => {
+  // 1234/AC2
+  const { client, server } = await serve([{ sha: COMMIT, tree: LISTING, truncated: true, page: 1, total_count: LISTING.length }]);
+  try {
+    await assert.rejects(client.commitTrees("novox", "hq", COMMIT), (err: Error) => {
+      assert.match(err.message, /novox\/hq/);
+      assert.ok(err.message.includes(COMMIT), err.message);
+      assert.match(err.message, /cut/);
+      return true;
+    });
+  } finally {
+    server.close();
+  }
+});
+
+test("a sha that is not a full commit, or no owner or repo, is refused before the forge is asked", async () => {
+  // 1234/AC3
+  const { getGiteaTools } = await import("../tools/index.ts");
+  const { client, server, asked } = await serve([]);
+  try {
+    const tool = getGiteaTools(client).find((t) => t.name === "gitea_commit_trees")!;
+    assert.ok(tool, "the tool is listed");
+    for (const args of [
+      { owner: "novox", repo: "hq", sha: "abc1234" },
+      { owner: "novox", repo: "hq", sha: "main" },
+      { owner: "novox", repo: "hq", sha: "" },
+      { owner: "novox", repo: "hq", sha: "c".repeat(41) },
+      { owner: "", repo: "hq", sha: COMMIT },
+      { owner: "novox", repo: " ", sha: COMMIT },
+    ]) await assert.rejects(tool.run(args as any), /./, JSON.stringify(args));
+    assert.equal(asked.length, 0, `the forge was asked ${asked.join(" ")}`);
+  } finally {
+    server.close();
+  }
+});
+
+test("gitea_open_pulls says each pull request's head branch beside its head sha", async () => {
+  // 1234/AC5
+  const { GiteaClient } = await import("../client.ts");
+  const server = createServer((req, res) => {
+    const url = req.url ?? "";
+    const send = (v: unknown) => res.end(JSON.stringify(v));
+    if (url.startsWith("/api/v1/repos/issues/search")) {
+      return send(url.includes("page=1") ? [{ number: 7, repository: { owner: "novox", name: "hq", full_name: "novox/hq" } }] : []);
+    }
+    if (url.startsWith("/api/v1/repos/novox/hq/pulls/7")) {
+      return send({ number: 7, state: "open", base: { ref: "main" }, head: { ref: "feat/x", sha: "aaa1111" }, html_url: "h" });
+    }
+    if (url.includes("/branch_protections")) { res.statusCode = 403; return res.end("{}"); }
+    if (url.startsWith("/api/v1/repos/novox/hq/branches/main")) return send({ name: "main" });
+    if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] });
+    res.statusCode = 404;
+    res.end("{}");
+  });
+  await new Promise<void>((r) => server.listen(0, r));
+  const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
+  try {
+    const [p] = await client.openPulls();
+    assert.deepEqual(p, { owner: "novox", repo: "hq", number: 7, base: "main", head: "feat/x", head_sha: "aaa1111", html_url: "h",
+      required: [], statuses: [], checks: {}, labels: [] });
+  } finally {
+    server.close();
+  }
+});
+
+test("the root tree is git's hash of the root's own entries, never the listing's top sha (the commit's)", async () => {
+  // 1234/AC6, from the live capture named in the header: novox/mesh-delivery at 7f61817.
+  const commit = "7f618177d6773d8d2d2b9e0b10e13b85ec700c59";
+  const listing = [
+    entry("README.md", "blob", "a087d22cbcbecce42256315671ff7e4d6ca85a7c"),
+    entry("merge-check.sh", "blob", "c4ef9a3dd6390b0edde7e4137095553897cecfb0"),
+    entry("modules", "tree", "31c0abb312ee6ce4693a7aacdd8b3fba871d6cf3"),
+    entry("modules/x", "tree", h("d")),
+    entry("modules/x/module.json", "blob", h("e")),
+  ];
+  const { GiteaClient } = await import("../client.ts");
+  const server = createServer((req, res) => {
+    res.end(JSON.stringify(new URL(req.url ?? "", "http://x").pathname.endsWith(`/git/trees/${commit}`)
+      ? { sha: commit, tree: listing, truncated: false, page: 1, total_count: listing.length } : {}));
+  });
+  await new Promise<void>((r) => server.listen(0, r));
+  const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
+  try {
+    const got = await client.commitTrees("novox", "mesh-delivery", commit);
+    assert.equal(got.root, "659bcaaac6c16972ec380267a4eba69d15558c26");
+    assert.deepEqual(got.trees, { "modules/x": h("d") });
+  } finally {
+    server.close();
+  }
+});

modules/gitea/test/token.test.ts

@@ -539,7 +539,7 @@ test("the tools register once there is a way to a token, and the first call mint
       "gitea_list_branches",
       "gitea_delete_branch",
       "gitea_branch_protection_get", "gitea_branch_protection_set",
-      "gitea_note_append", "gitea_delivery_view", "gitea_commit_statuses", "gitea_open_pulls", "gitea_contains",
+      "gitea_note_append", "gitea_delivery_view", "gitea_commit_statuses", "gitea_open_pulls", "gitea_commit_trees", "gitea_contains",
       "gitea_list_labels", "gitea_create_label",
       "gitea_api",
     ],

modules/gitea/tools/index.ts

@@ -523,6 +523,22 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
       input: {},
       run: async () => ({ pulls: await gitea.openPulls() }),
     },
+    {
+      name: "gitea_commit_trees",
+      description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.",
+      input: {
+        owner: { type: "string", description: "the repository owner" },
+        repo: { type: "string", description: "the repository name" },
+        sha: { type: "string", description: "the commit, its full 40-hex sha" },
+      },
+      run: async (args) => {
+        const owner = String(args.owner ?? "").trim(), repo = String(args.repo ?? "").trim();
+        const sha = String(args.sha ?? "").trim();
+        if (!owner || !repo) throw new Error("name the repository: owner and repo");
+        if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error("sha is a commit's full 40-hex sha");
+        return gitea.commitTrees(owner, repo, sha);
+      },
+    },
     {
       name: "gitea_contains",
       description: "Whether a commit is contained in another — an ancestor of it, or the same — from the forge's compare of the two: what mesh-delivery reads before it says a held merge is covered by a later one (novox/hq ADR 0270). Commits only, by sha or the start of one. Reads only.",