From 3249b9a0cc8b22731a44717a0ee5f29280834fa6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 23 Sep 2026 23:19:12 +0200 Subject: [PATCH] The registry's public name is a second module beside the store, locked by the registry itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The predecessor serves the registry under a public name, behind htpasswd basic auth, with a twenty-gigabyte body limit for layer pushes. The mesh's registry has no name, no lock and no limit — by design inside the mesh, where the private network is the boundary and every node pulls without an account (hq ADR 0082). Taking the name over must not change that. A route on `distribution` itself would: contributing a route is requiring one, and the store is raised at genesis on a node with no proxy. So the public door is `distribution-gate`, a second registry process on the same volume, behind the registry's own htpasswd (the predecessor's realm, the predecessor's file, carried in with `secret accept`), with the route and its limit. It requires the store's storage as a node-scoped provision, so it can only land beside the store. The store's own door is untouched — no auth, no htpasswd — which is what keeps the builder's pushes and every node's pulls working. Both processes read the predecessor's configuration where it changed behaviour: delete enabled, which tag retention depends on; no per-process descriptor cache, which two processes over one store cannot share; the CORS headers for the retired interface dropped. route-adapter writes the limit as the predecessor's own buffering middleware, named after the router, only when asked for — and skips a route whose limit it cannot read rather than carrying what the module said not to. hq ADR 0082/0104, the registry hand-over. --- modules/distribution-gate/module.json | 62 ++++++++++++++++++++ modules/distribution/module.json | 20 ++++++- modules/route-adapter/README.md | 24 +++++++- modules/route-adapter/adapter.ts | 53 ++++++++++++++++- modules/route-adapter/test/adapter.test.ts | 67 +++++++++++++++++++++- 5 files changed, 220 insertions(+), 6 deletions(-) create mode 100644 modules/distribution-gate/module.json diff --git a/modules/distribution-gate/module.json b/modules/distribution-gate/module.json new file mode 100644 index 0000000..9cca3f9 --- /dev/null +++ b/modules/distribution-gate/module.json @@ -0,0 +1,62 @@ +{ + "module": "distribution-gate", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "requires": [ + "artifact-storage", + "route" + ], + "contributes": { + "route": { + "label": "registry-api", + "port": 5001, + "max-request-body": 21474836480 + } + }, + "own-secrets": { + "htpasswd": "/var/lib/mesh/registry-gate/htpasswd" + }, + "listens": [ + { + "port": 5001, + "protocol": "tcp", + "from": "mesh", + "why": "the artifact store's public door: the same store behind the registry's own basic auth, reached by the proxy under its public name; the mesh itself pulls from the store's own port and never from here" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "path": "/var/lib/mesh/registry-gate", + "mode": "0700" + }, + { + "id": "config", + "type": "file", + "path": "/var/lib/mesh/registry-gate/config.yml", + "mode": "0644", + "content": "# The registry's configuration, written by the mesh from the module's manifest.\n#\n# Carried over from the predecessor's registry.yml where it changed behaviour (novox/hq ADR 0082,\n# the registry hand-over):\n# - storage.delete.enabled: the image's default refuses DELETE on a manifest; the predecessor\n# enabled it, and tag retention and garbage collection depend on it.\n# - no storage.cache: the image's default keeps an in-memory blob-descriptor cache, which is\n# right for one process and wrong for two on one store — the mesh door and the public door\n# are two registry processes sharing this filesystem, and a descriptor cached by one and\n# deleted through the other would say a blob exists that does not.\n# Dropped: the CORS headers, which served the browser interface that is being retired.\nversion: 0.1\nlog:\n fields:\n service: registry\nstorage:\n delete:\n enabled: true\n filesystem:\n rootdirectory: /var/lib/registry\nhttp:\n addr: :5001\n headers:\n X-Content-Type-Options: [nosniff]\nhealth:\n storagedriver:\n enabled: true\n interval: 10s\n threshold: 3\n# The public door's lock, as the predecessor kept it: the registry itself checks basic auth\n# against an htpasswd file — bcrypt entries, one user — under the realm the predecessor\n# announced, so a client that logged in to the old name logs in to this one unchanged.\nauth:\n htpasswd:\n realm: basic-realm\n path: /etc/docker/registry/htpasswd\n" + }, + { + "id": "gate", + "type": "container", + "name": "mesh-registry-gate", + "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "ports": [ + "5001:5001" + ], + "volumes": [ + "mesh-registry-data:/var/lib/registry", + "/var/lib/mesh/registry-gate/config.yml:/etc/docker/registry/config.yml:ro", + "/var/lib/mesh/registry-gate/htpasswd:/etc/docker/registry/htpasswd:ro" + ], + "restart-on": [ + "config", + "needs-htpasswd" + ] + } + ] +} diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 029fb04..ceb7b0f 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -5,6 +5,10 @@ { "name": "artifact-store", "scope": "mesh" + }, + { + "name": "artifact-storage", + "scope": "node" } ], "claims": [ @@ -25,6 +29,9 @@ "serves": { "artifact-store": { "port": 5000 + }, + "artifact-storage": { + "volume": "mesh-registry-data" } }, "listens": [ @@ -42,6 +49,13 @@ "path": "/var/lib/mesh/registry", "mode": "0700" }, + { + "id": "config", + "type": "file", + "path": "/var/lib/mesh/registry/config.yml", + "mode": "0644", + "content": "# The registry's configuration, written by the mesh from the module's manifest.\n#\n# Carried over from the predecessor's registry.yml where it changed behaviour (novox/hq ADR 0082,\n# the registry hand-over):\n# - storage.delete.enabled: the image's default refuses DELETE on a manifest; the predecessor\n# enabled it, and tag retention and garbage collection depend on it.\n# - no storage.cache: the image's default keeps an in-memory blob-descriptor cache, which is\n# right for one process and wrong for two on one store — the mesh door and the public door\n# are two registry processes sharing this filesystem, and a descriptor cached by one and\n# deleted through the other would say a blob exists that does not.\n# Dropped: the CORS headers, which served the browser interface that is being retired.\nversion: 0.1\nlog:\n fields:\n service: registry\nstorage:\n delete:\n enabled: true\n filesystem:\n rootdirectory: /var/lib/registry\nhttp:\n addr: :5000\n headers:\n X-Content-Type-Options: [nosniff]\nhealth:\n storagedriver:\n enabled: true\n interval: 10s\n threshold: 3\n" + }, { "id": "store", "type": "container", @@ -51,7 +65,11 @@ "5000:5000" ], "volumes": [ - "mesh-registry-data:/var/lib/registry" + "mesh-registry-data:/var/lib/registry", + "/var/lib/mesh/registry/config.yml:/etc/docker/registry/config.yml:ro" + ], + "restart-on": [ + "config" ] } ] diff --git a/modules/route-adapter/README.md b/modules/route-adapter/README.md index ccf82c2..efa2fe1 100644 --- a/modules/route-adapter/README.md +++ b/modules/route-adapter/README.md @@ -56,12 +56,31 @@ http: - url: http://host.docker.internal:2999 ``` +A contribution may also carry **`max-request-body`**, the largest request body in bytes the proxy +may carry to it — the registry's public name needs twenty gigabytes for a layer push (novox/hq +ADR 0082, the registry hand-over). It is written as the predecessor's own `buffering` middleware, +named after the router, and only when asked for: + +```yaml + routers: + mesh-registry-api-example: + # … + middlewares: [mesh-registry-api-example-body] + middlewares: + mesh-registry-api-example-body: + buffering: + maxRequestBodyBytes: 21474836480 +``` + - **The certificate resolver is the predecessor's own**, by its own name. The predecessor already holds a certificate for every public name it serves, so naming its resolver means a migrated name is served from the certificate that exists. A resolver of the mesh's would ask a public authority for one in the same window the module is cut over — the risk ADR 0104 exists to remove. - **The port is the contributor's**, straight out of the contribution: the mesh assigned the machine-side number (novox/hq ADR 0038) and carries it there. Nothing here guesses it. +- **A limit it cannot honour is a route it does not write.** A `max-request-body` that is not a + whole positive number of bytes is skipped and named, like a port that is not one — written + without the limit, the predecessor would carry exactly what the module said not to carry. - **The address is where the mesh says that machine is.** Empty means this one, reached from inside the predecessor's container at `host.docker.internal` — not loopback, which from inside that container is the container. A contributor on another node carries its overlay address and the @@ -157,5 +176,6 @@ cd modules/route-adapter && npm test ``` They hold it to what ADR 0104 says holds it: one file per contribution, a file removed when its -contribution goes, every file it did not write left alone — and the two facts a route file has to -get right, the port the contributor publishes and the address of the machine it is on. +contribution goes, every file it did not write left alone — and the three facts a route file has to +get right: the port the contributor publishes, the address of the machine it is on, and the body +limit it asked for, written as the predecessor's middleware or not at all. diff --git a/modules/route-adapter/adapter.ts b/modules/route-adapter/adapter.ts index 9b0324a..c4581df 100644 --- a/modules/route-adapter/adapter.ts +++ b/modules/route-adapter/adapter.ts @@ -75,6 +75,16 @@ export interface Route { from: string; /** Where the predecessor's proxy is to send it. */ target: string; + /** + * The largest request body, in bytes, the predecessor may carry to it — the contribution's + * `max-request-body`. Absent is whatever the predecessor does by default. + * + * **The registry's hand-over is why it exists** (novox/hq ADR 0082). A registry takes image + * layers in single requests of gigabytes, and the predecessor served the registry's public name + * with exactly this as a buffering middleware; a route that could not say it would have a public + * name it could not be pushed to. + */ + maxRequestBody?: number; } /** What one pass changed. */ @@ -156,10 +166,22 @@ export function routesFrom(document: unknown, machine: string): { routes: Route[ skipped.push(`${from} asked for route ${name} and gave no usable port`); continue; } + // A limit it cannot honour is a route it does not write — skipped and named, like a port that + // is not one. Written without the limit instead, the predecessor would carry exactly what the + // module said not to carry, and this module would report success. + const asked_limit = entry.values?.["max-request-body"]; + const limit = asBodyLimit(asked_limit); + if (limit === null) { + skipped.push( + `${from} asked for route ${name} with a max-request-body of ${JSON.stringify(asked_limit)}, ` + + `which is not a whole positive number of bytes`, + ); + continue; + } // Where the mesh says that machine is. Empty means this one, and this one is reached from // inside the predecessor's container by the machine's own name, not by loopback. const at = typeof entry.at === "string" && entry.at.trim() !== "" ? entry.at.trim() : machine; - routes.push({ name, from, target: `http://${at}:${port}` }); + routes.push({ name, from, target: `http://${at}:${port}`, ...(limit === undefined ? {} : { maxRequestBody: limit }) }); } return { routes, skipped }; } @@ -183,6 +205,10 @@ export function routerNameFor(name: string): string { */ export function routeFile(route: Route, settings: Settings): string { const id = routerNameFor(route.name); + // The body limit is a middleware in the predecessor's vocabulary — its `buffering`, with the + // one field the predecessor's own registry route set — named after the router so the two halves + // cannot drift, and written only when the contribution asked for it. + const limited = route.maxRequestBody !== undefined; return [ marker, `# ${route.from} contributed this route. It is removed when that contribution goes.`, @@ -192,10 +218,19 @@ export function routeFile(route: Route, settings: Settings): string { ` entryPoints: [${settings.entrypoint}]`, ` rule: Host(\`${route.name}\`)`, ` service: ${id}`, + ...(limited ? [` middlewares: [${id}-body]`] : []), " tls:", ` certResolver: ${settings.resolver}`, " domains:", ` - main: ${route.name}`, + ...(limited + ? [ + " middlewares:", + ` ${id}-body:`, + " buffering:", + ` maxRequestBodyBytes: ${route.maxRequestBody}`, + ] + : []), " services:", ` ${id}:`, " loadBalancer:", @@ -279,6 +314,22 @@ export async function reconcile(routes: Route[], settings: Settings): Promise = {}): Promise { + const settings = await predecessor(); + const changed = await pass(settings, contributed( + { from: "distribution-gate", name: "registry-api.example", port: 5001, limit: 21474836480 }, + { from: "gitea", name: "git.example", port: 2999 }, + )); + assert.deepEqual(changed.written, ["mesh-git.example.yml", "mesh-registry-api.example.yml"]); + + assert.equal(await readFile(join(settings.dynamic, "mesh-registry-api.example.yml"), "utf8"), [ + marker, + "# distribution-gate contributed this route. It is removed when that contribution goes.", + "http:", + " routers:", + " mesh-registry-api-example:", + " entryPoints: [websecure]", + " rule: Host(`registry-api.example`)", + " service: mesh-registry-api-example", + " middlewares: [mesh-registry-api-example-body]", + " tls:", + " certResolver: le", + " domains:", + " - main: registry-api.example", + " middlewares:", + " mesh-registry-api-example-body:", + " buffering:", + " maxRequestBodyBytes: 21474836480", + " services:", + " mesh-registry-api-example:", + " loadBalancer:", + " servers:", + " - url: http://host.docker.internal:5001", + "", + ].join("\n")); + + // The route that asked for nothing carries no middleware — the predecessor's default stands. + const plain = await readFile(join(settings.dynamic, "mesh-git.example.yml"), "utf8"); + assert.doesNotMatch(plain, /middlewares|buffering/); +}); + +// A limit it cannot honour is a route it does not write. Written without it, the predecessor would +// carry exactly what the module said not to carry, and this module would report success. +test("a body limit that is not a number of bytes is skipped and named", () => { + const machine = defaults.machine; + for (const limit of ["20g", 0, -1, 1.5, true, null]) { + const { routes, skipped } = routesFrom( + contributed({ from: "gate", name: "registry-api.example", port: 5001, limit }), machine); + assert.deepEqual(routes, [], `a limit of ${JSON.stringify(limit)} was served`); + assert.equal(skipped.length, 1); + assert.match(skipped[0]!, /max-request-body/); + } + // And a limit the controller would accept is carried, as a number. + const { routes } = routesFrom( + contributed({ from: "gate", name: "registry-api.example", port: 5001, limit: 1024 }), machine); + assert.equal(routes[0]?.maxRequestBody, 1024); +});