Merge pull request 'The catalogue's check uses the shared binary refusal, commit by commit (issue 1577)' (#508) from fix/1577-checker-refuses-binaries into main

modules/gitea/client.ts

@@ -174,8 +174,27 @@ function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
   console.error(`the setting ${key} is ignored: it comes from the vault`);
 }
 
+/** How long the forge may take to answer one request before it is cut and said (novox/hq issue 1577: a request for a 5 MB
+ *  file never ended, and every read through the module timed out behind it). */
+export const FORGE_REQUEST_TIMEOUT_MS = 20_000;
+
+/** The most a tool's answer carries of one file: the bus's message bound is 1 MiB and a file travels as text (issue 1577). */
+export const MAX_FILE_BYTES = 512 * 1024;
+
+/** One request to the forge, cut at the limit and failing loud naming the request. */
+async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise<Response> {
+  try {
+    return await fetch(url, { ...init, signal: init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs) });
+  } catch (err) {
+    if (err instanceof Error && err.name === "TimeoutError") throw new Error(`Gitea API ${label}: no answer in ${timeoutMs / 1000} s, the request was cut`);
+    throw err;
+  }
+}
+
 export class GiteaClient {
   readonly baseUrl: string;
+  /** The limit of one request; a test sets it short. */
+  requestTimeoutMs = FORGE_REQUEST_TIMEOUT_MS;
   private readonly tokens: TokenSource;
 
   /** A token given as a string is one somebody configured; a source decides for itself (token.ts). */
@@ -230,14 +249,14 @@ export class GiteaClient {
 
   private send(path: string, options: RequestInit, token: string): Promise<Response> {
     checkSentPath(path);
-    return fetch(`${this.baseUrl}/api/v1${path}`, {
+    return timedFetch(`${this.baseUrl}/api/v1${path}`, {
       ...options,
       headers: {
         "Content-Type": "application/json",
         Authorization: `token ${token}`,
         ...(options.headers as Record<string, string> | undefined),
       },
-    });
+    }, `${options.method ?? "GET"} ${path}`, this.requestTimeoutMs);
   }
 
   /** Generic authenticated API call — the escape hatch for endpoints without a dedicated method.
@@ -771,6 +790,22 @@ export class GiteaClient {
   /** One file's contents at a ref (default the repository's default branch), decoded. */
   async getFile(owner: string, repo: string, path: string, ref?: string): Promise<{ path: string; ref?: string; sha: string; size: number; content: string }> {
     const qs = ref ? `?ref=${encodeURIComponent(ref)}` : "";
+    // The size first, from the file's directory listing, which carries no content: a file over what an answer can carry is
+    // refused without its body being fetched (issue 1577). A file the listing does not show is asked for as before.
+    const clean = path.replace(/^\/+|\/+$/g, "");
+    pathSegments("file path", clean); // a path that is another path or user is refused before anything reaches the forge
+    const slash = clean.lastIndexOf("/");
+    const dir = slash < 0 ? "" : clean.slice(0, slash);
+    let listing: any[] | null = null;
+    try {
+      listing = await this.request<any[]>(`${repoPath(owner, repo)}/contents${dir ? "/" + pathSegments("file path", dir) : ""}${qs}`);
+    } catch (err) {
+      if (!(err instanceof Error) || !/: 404 /.test(err.message)) throw err;
+    }
+    const entry = Array.isArray(listing) ? listing.find((e) => e?.path === clean) : undefined;
+    if (entry && Number(entry.size) > MAX_FILE_BYTES) {
+      throw new Error(`the file is ${Number(entry.size)} bytes, over the ${MAX_FILE_BYTES / 1024} KiB a tool answer can carry; read it from a clone`);
+    }
     const f = await this.request<any>(`${repoPath(owner, repo)}/contents/${pathSegments("file path", path.replace(/^\/+|\/+$/g, ""))}${qs}`);
     if (!f || f.type !== "file") throw new Error(`Gitea API: ${path} is not a file`);
     const content = f.encoding === "base64" ? Buffer.from(String(f.content ?? ""), "base64").toString("utf8") : String(f.content ?? "");
@@ -940,14 +975,14 @@ export class GiteaAdmin {
    *  throwing on it — the caller decides which non-2xx codes are idempotent successes. Only an
    *  unexpected status becomes an error, and only where the caller says so. */
   private async request(path: string, options: RequestInit = {}): Promise<AdminResponse> {
-    const res = await fetch(`${this.baseUrl}/api/v1${path}`, {
+    const res = await timedFetch(`${this.baseUrl}/api/v1${path}`, {
       ...options,
       headers: {
         "Content-Type": "application/json",
         Authorization: this.authorization,
         ...(options.headers as Record<string, string> | undefined),
       },
-    });
+    }, `${options.method ?? "GET"} ${path}`, FORGE_REQUEST_TIMEOUT_MS);
     const text = await res.text();
     let body: any = null;
     if (text) {
@@ -1192,9 +1227,9 @@ export class GiteaAdmin {
    * other answer rejects (novox/hq issue 120).
    */
   async holdsTeamMember(org: string, team: string, username: string, password: string): Promise<boolean> {
-    const me = await fetch(`${this.baseUrl}/api/v1/user`, {
+    const me = await timedFetch(`${this.baseUrl}/api/v1/user`, {
       headers: { Authorization: "Basic " + Buffer.from(`${username}:${password}`).toString("base64") },
-    });
+    }, "GET /user", FORGE_REQUEST_TIMEOUT_MS);
     if (me.status === 401 || me.status === 403) return false;
     if (me.status !== 200) throw new Error(`Gitea GET /user as ${username}: ${me.status}`);
     const teams = await this.request(`/orgs/${encodeURIComponent(org)}/teams?limit=50`);

modules/gitea/test/guard.test.ts

@@ -640,6 +640,7 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud
   await tool("gitea_delete_branch").run({ ...repo, branch: "feat/x" });
   await tool("gitea_get_file").run({ ...repo, path: "modules/gitea/module.json" }).catch(() => undefined);
   assert.deepEqual(state.asked, ["DELETE /repos/novox/mesh-catalog/branches/feat/x",
+    "GET /repos/novox/mesh-catalog/contents/modules/gitea", // the file's size first (issue 1577)
     "GET /repos/novox/mesh-catalog/contents/modules/gitea/module.json"]);
 });
 

modules/gitea/test/paging.test.ts

@@ -90,3 +90,48 @@ test("a pass pages until it reaches back past its floor, and no further", async
     server.close();
   }
 });
+
+// **One forge request never holds the others, and a file too big for an answer is not downloaded** (novox/hq issue 1577): a
+// request for a 4.9 MB file never ended and every read through the module timed out behind it.
+test("a request the forge never answers is cut at the limit and says which request", async () => {
+  const { GiteaClient } = await import("../client.ts");
+  const server = createServer((req) => { if (!req.url?.includes("fast")) return; });
+  await new Promise<void>((r) => server.listen(0, r));
+  const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
+  client.requestTimeoutMs = 150;
+  try {
+    const started = Date.now();
+    await assert.rejects(client.api("/repos/novox/hq/pulls"), /Gitea API GET \/repos\/novox\/hq\/pulls: no answer in 0\.15 s/);
+    assert.ok(Date.now() - started < 3000);
+  } finally {
+    server.closeAllConnections();
+    await new Promise<void>((r) => server.close(() => r()));
+  }
+});
+
+test("a file over 512 KiB is refused from its directory listing, and its body is never asked for", async () => {
+  const { GiteaClient } = await import("../client.ts");
+  const asked: string[] = [];
+  const server = createServer((req, res) => {
+    asked.push(req.url ?? "");
+    res.setHeader("Content-Type", "application/json");
+    if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin?")) {
+      return res.end(JSON.stringify([{ path: "bin/big", type: "file", size: 5_000_000 }, { path: "bin/small", type: "file", size: 5 }]));
+    }
+    if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin/small")) {
+      return res.end(JSON.stringify({ type: "file", encoding: "base64", sha: "s", size: 5, content: Buffer.from("hello").toString("base64") }));
+    }
+    res.statusCode = 404;
+    res.end("{}");
+  });
+  await new Promise<void>((r) => server.listen(0, r));
+  const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
+  try {
+    await assert.rejects(client.getFile("novox", "hq", "bin/big", "main"),
+      /the file is 5000000 bytes, over the 512 KiB a tool answer can carry; read it from a clone/);
+    assert.ok(!asked.some((u) => u.includes("/contents/bin/big")), asked.join("\n"));
+    assert.equal((await client.getFile("novox", "hq", "bin/small", "main")).content, "hello");
+  } finally {
+    await new Promise<void>((r) => server.close(() => r()));
+  }
+});

modules/gitea/tools/index.ts

@@ -374,7 +374,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
     // ---- Contents and branches ----
     {
       name: "gitea_get_file",
-      description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch).",
+      description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch). Refuses a file over 512 KiB, which a tool answer cannot carry: read it from a clone.",
       input: {
         owner: { type: "string", description: "the repository owner" },
         repo: { type: "string", description: "the repository name" },

modules/mesh-build-go/cmd/mesh-build-go/fetch.go

@@ -42,11 +42,13 @@ type FetchResult struct {
 	// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
 	// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
 	// files, and the workspace is left at the trunk commit.
-	Base      string                          `json:"base,omitempty"`
+	Base      string   `json:"base,omitempty"`
-	Merge     string                          `json:"merge,omitempty"`
+	Merge     string   `json:"merge,omitempty"`
-	Conflicts []string                        `json:"conflicts,omitempty"`
+	Conflicts []string `json:"conflicts,omitempty"`
-	Changed   []string                        `json:"changed,omitempty"`
+	Changed   []string `json:"changed,omitempty"`
-	Trees     map[string]buildstep.ModuleTree `json:"trees,omitempty"`
+	// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
+	Programs *[]buildstep.Program            `json:"programs,omitempty"`
+	Trees    map[string]buildstep.ModuleTree `json:"trees,omitempty"`
 	// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
 	// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
 	Beside []Sibling `json:"beside,omitempty"`

modules/mesh-build-go/cmd/mesh-build-go/preview.go

@@ -41,6 +41,9 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base
 	}
 	res.Base = mr.Base
 	res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
+	if mr.Merge != "" {
+		res.Programs = &mr.Programs
+	}
 	if res.Merge == "" {
 		return nil
 	}

modules/mesh-build-go/cmd/mesh-build-go/preview_test.go

@@ -77,7 +77,7 @@ func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
 		t.Fatal(err)
 	}
 	if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 ||
-		strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" {
+		strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" || res.Programs == nil || len(*res.Programs) != 0 { // 1577: a merge says it found no program
 		t.Errorf("result = %s", st.Result)
 	}
 	mt, ok := res.Trees["modules/m"]

modules/mesh-build-go/go.mod

@@ -3,7 +3,7 @@ module meshbuildgo
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

modules/mesh-build-go/go.sum

@@ -1,5 +1,9 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 h1:ozuxlh0fOlq1c3mJ4DLNPgQQ+AIaqd6nWd5CQaEe/r4=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=

modules/mesh-build-image/cmd/mesh-build-image/fetch.go

@@ -31,13 +31,15 @@ const ForgeAlias = "mesh-forge"
 // files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
 // merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is left at the trunk.
 type FetchResult struct {
-	Workspace string                          `json:"workspace"`
+	Workspace string   `json:"workspace"`
-	Commit    string                          `json:"commit"`
+	Commit    string   `json:"commit"`
-	Base      string                          `json:"base,omitempty"`
+	Base      string   `json:"base,omitempty"`
-	Merge     string                          `json:"merge,omitempty"`
+	Merge     string   `json:"merge,omitempty"`
-	Conflicts []string                        `json:"conflicts,omitempty"`
+	Conflicts []string `json:"conflicts,omitempty"`
-	Changed   []string                        `json:"changed,omitempty"`
+	Changed   []string `json:"changed,omitempty"`
-	Trees     map[string]buildstep.ModuleTree `json:"trees,omitempty"`
+	// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
+	Programs *[]buildstep.Program            `json:"programs,omitempty"`
+	Trees    map[string]buildstep.ModuleTree `json:"trees,omitempty"`
 }
 
 func (m *Module) fetchTool() StepTool {
@@ -127,6 +129,9 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base
 	}
 	res.Base = mr.Base
 	res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
+	if mr.Merge != "" {
+		res.Programs = &mr.Programs
+	}
 	if res.Merge == "" {
 		return nil
 	}

modules/mesh-build-image/go.mod

@@ -3,7 +3,7 @@ module meshbuildimage
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-build-image/go.sum

@@ -1,5 +1,9 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 h1:ozuxlh0fOlq1c3mJ4DLNPgQQ+AIaqd6nWd5CQaEe/r4=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-build-node/cmd/mesh-build-node/fetch.go

@@ -51,11 +51,13 @@ type FetchResult struct {
 	// commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory.
 	// A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is
 	// left at the trunk commit with nothing installed.
-	Base      string                          `json:"base,omitempty"`
+	Base      string   `json:"base,omitempty"`
-	Merge     string                          `json:"merge,omitempty"`
+	Merge     string   `json:"merge,omitempty"`
-	Conflicts []string                        `json:"conflicts,omitempty"`
+	Conflicts []string `json:"conflicts,omitempty"`
-	Changed   []string                        `json:"changed,omitempty"`
+	Changed   []string `json:"changed,omitempty"`
-	Trees     map[string]buildstep.ModuleTree `json:"trees,omitempty"`
+	// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
+	Programs *[]buildstep.Program            `json:"programs,omitempty"`
+	Trees    map[string]buildstep.ModuleTree `json:"trees,omitempty"`
 	// Group is the group's package the fetch packed and served to the TypeScript members (the call's `package`):
 	// "<name>@<version> <integrity>".
 	Group string `json:"group,omitempty"`

modules/mesh-build-node/cmd/mesh-build-node/preview.go

@@ -27,6 +27,9 @@ func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResul
 	}
 	res.Base = mr.Base
 	res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
+	if mr.Merge != "" {
+		res.Programs = &mr.Programs
+	}
 	if res.Merge == "" {
 		return nil
 	}

modules/mesh-build-node/go.mod

@@ -3,7 +3,7 @@ module meshbuildnode
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-build-node/go.sum

@@ -1,5 +1,9 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f h1:LwefZSIEVjhdhgx22kgL+eJFfD9VVGJblYNPUCh1yZQ=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026175935-22bc4dc8ef4f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397 h1:ozuxlh0fOlq1c3mJ4DLNPgQQ+AIaqd6nWd5CQaEe/r4=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184717-fe11db929397/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=