novox/mesh-catalog
3
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

The registry's public name is a second module beside the store, locked by the registry itself

Browse Source 
The predecessor serves the registry under a public name, behind htpasswd basic auth, with a
twenty-gigabyte body limit for layer pushes. The mesh's registry has no name, no lock and no
limit — by design inside the mesh, where the private network is the boundary and every node
pulls without an account (hq ADR 0082). Taking the name over must not change that.

A route on `distribution` itself would: contributing a route is requiring one, and the store
is raised at genesis on a node with no proxy. So the public door is `distribution-gate`, a
second registry process on the same volume, behind the registry's own htpasswd (the
predecessor's realm, the predecessor's file, carried in with `secret accept`), with the
route and its limit. It requires the store's storage as a node-scoped provision, so it can
only land beside the store. The store's own door is untouched — no auth, no htpasswd — which
is what keeps the builder's pushes and every node's pulls working.

Both processes read the predecessor's configuration where it changed behaviour: delete
enabled, which tag retention depends on; no per-process descriptor cache, which two
processes over one store cannot share; the CORS headers for the retired interface dropped.

route-adapter writes the limit as the predecessor's own buffering middleware, named after
the router, only when asked for — and skips a route whose limit it cannot read rather than
carrying what the module said not to.

hq ADR 0082/0104, the registry hand-over.
This commit is contained in:
jschoubben
2026-09-23 23:19:12 +02:00
parent 9f2c678355
commit 3249b9a0cc
5 changed files with 220 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/distribution-gate/module.json
+62
View File
@@ -0,0 +1,62 @@
{
"module": "distribution-gate",
"version": "1",
"capabilities": [
"container-runtime"
],
"requires": [
"artifact-storage",
"route"
],
"contributes": {
"route": {
"label": "registry-api",
"port": 5001,
"max-request-body": 21474836480
}
},
"own-secrets": {
"htpasswd": "/var/lib/mesh/registry-gate/htpasswd"
},
"listens": [
{
"port": 5001,
"protocol": "tcp",
"from": "mesh",
"why": "the artifact store's public door: the same store behind the registry's own basic auth, reached by the proxy under its public name; the mesh itself pulls from the store's own port and never from here"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"path": "/var/lib/mesh/registry-gate",
"mode": "0700"
},
{
"id": "config",
"type": "file",
"path": "/var/lib/mesh/registry-gate/config.yml",
"mode": "0644",
"content": "# The registry's configuration, written by the mesh from the module's manifest.\n#\n# Carried over from the predecessor's registry.yml where it changed behaviour (novox/hq ADR 0082,\n# the registry hand-over):\n# - storage.delete.enabled: the image's default refuses DELETE on a manifest; the predecessor\n# enabled it, and tag retention and garbage collection depend on it.\n# - no storage.cache: the image's default keeps an in-memory blob-descriptor cache, which is\n# right for one process and wrong for two on one store — the mesh door and the public door\n# are two registry processes sharing this filesystem, and a descriptor cached by one and\n# deleted through the other would say a blob exists that does not.\n# Dropped: the CORS headers, which served the browser interface that is being retired.\nversion: 0.1\nlog:\n fields:\n service: registry\nstorage:\n delete:\n enabled: true\n filesystem:\n rootdirectory: /var/lib/registry\nhttp:\n addr: :5001\n headers:\n X-Content-Type-Options: [nosniff]\nhealth:\n storagedriver:\n enabled: true\n interval: 10s\n threshold: 3\n# The public door's lock, as the predecessor kept it: the registry itself checks basic auth\n# against an htpasswd file — bcrypt entries, one user — under the realm the predecessor\n# announced, so a client that logged in to the old name logs in to this one unchanged.\nauth:\n htpasswd:\n realm: basic-realm\n path: /etc/docker/registry/htpasswd\n"
},
{
"id": "gate",
"type": "container",
"name": "mesh-registry-gate",
"image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373",
"ports": [
"5001:5001"
],
"volumes": [
"mesh-registry-data:/var/lib/registry",
"/var/lib/mesh/registry-gate/config.yml:/etc/docker/registry/config.yml:ro",
"/var/lib/mesh/registry-gate/htpasswd:/etc/docker/registry/htpasswd:ro"
],
"restart-on": [
"config",
"needs-htpasswd"
]
}
]
}
modules/distribution/module.json
+19 -1
View File
@@ -5,6 +5,10 @@
{
"name": "artifact-store",
"scope": "mesh"
},
{
"name": "artifact-storage",
"scope": "node"
}
],
"claims": [
@@ -25,6 +29,9 @@
"serves": {
"artifact-store": {
"port": 5000
},
"artifact-storage": {
"volume": "mesh-registry-data"
}
},
"listens": [
@@ -42,6 +49,13 @@
"path": "/var/lib/mesh/registry",
"mode": "0700"
},
{
"id": "config",
"type": "file",
"path": "/var/lib/mesh/registry/config.yml",
"mode": "0644",
"content": "# The registry's configuration, written by the mesh from the module's manifest.\n#\n# Carried over from the predecessor's registry.yml where it changed behaviour (novox/hq ADR 0082,\n# the registry hand-over):\n# - storage.delete.enabled: the image's default refuses DELETE on a manifest; the predecessor\n# enabled it, and tag retention and garbage collection depend on it.\n# - no storage.cache: the image's default keeps an in-memory blob-descriptor cache, which is\n# right for one process and wrong for two on one store — the mesh door and the public door\n# are two registry processes sharing this filesystem, and a descriptor cached by one and\n# deleted through the other would say a blob exists that does not.\n# Dropped: the CORS headers, which served the browser interface that is being retired.\nversion: 0.1\nlog:\n fields:\n service: registry\nstorage:\n delete:\n enabled: true\n filesystem:\n rootdirectory: /var/lib/registry\nhttp:\n addr: :5000\n headers:\n X-Content-Type-Options: [nosniff]\nhealth:\n storagedriver:\n enabled: true\n interval: 10s\n threshold: 3\n"
},
{
"id": "store",
"type": "container",
@@ -51,7 +65,11 @@
"5000:5000"
],
"volumes": [
"mesh-registry-data:/var/lib/registry"
"mesh-registry-data:/var/lib/registry",
"/var/lib/mesh/registry/config.yml:/etc/docker/registry/config.yml:ro"
],
"restart-on": [
"config"
]
}
]
modules/route-adapter/README.md
+22 -2
View File
@@ -56,12 +56,31 @@ http:
- url: http://host.docker.internal:2999
```
A contribution may also carry **`max-request-body`**, the largest request body in bytes the proxy
may carry to it — the registry's public name needs twenty gigabytes for a layer push (novox/hq
ADR 0082, the registry hand-over). It is written as the predecessor's own `buffering` middleware,
named after the router, and only when asked for:
```yaml
routers:
mesh-registry-api-example:
# …
middlewares: [mesh-registry-api-example-body]
middlewares:
mesh-registry-api-example-body:
buffering:
maxRequestBodyBytes: 21474836480
```
- **The certificate resolver is the predecessor's own**, by its own name. The predecessor already
holds a certificate for every public name it serves, so naming its resolver means a migrated name
is served from the certificate that exists. A resolver of the mesh's would ask a public authority
for one in the same window the module is cut over — the risk ADR 0104 exists to remove.
- **The port is the contributor's**, straight out of the contribution: the mesh assigned the
machine-side number (novox/hq ADR 0038) and carries it there. Nothing here guesses it.
- **A limit it cannot honour is a route it does not write.** A `max-request-body` that is not a
whole positive number of bytes is skipped and named, like a port that is not one — written
without the limit, the predecessor would carry exactly what the module said not to carry.
- **The address is where the mesh says that machine is.** Empty means this one, reached from inside
the predecessor's container at `host.docker.internal` — not loopback, which from inside that
container is the container. A contributor on another node carries its overlay address and the
@@ -157,5 +176,6 @@ cd modules/route-adapter && npm test
```
They hold it to what ADR 0104 says holds it: one file per contribution, a file removed when its
contribution goes, every file it did not write left alone — and the two facts a route file has to
get right, the port the contributor publishes and the address of the machine it is on.
contribution goes, every file it did not write left alone — and the three facts a route file has to
get right: the port the contributor publishes, the address of the machine it is on, and the body
limit it asked for, written as the predecessor's middleware or not at all.
modules/route-adapter/adapter.ts
+52 -1
View File
@@ -75,6 +75,16 @@ export interface Route {
from: string;
/** Where the predecessor's proxy is to send it. */
target: string;
/**
* The largest request body, in bytes, the predecessor may carry to it — the contribution's
* `max-request-body`. Absent is whatever the predecessor does by default.
*
* **The registry's hand-over is why it exists** (novox/hq ADR 0082). A registry takes image
* layers in single requests of gigabytes, and the predecessor served the registry's public name
* with exactly this as a buffering middleware; a route that could not say it would have a public
* name it could not be pushed to.
*/
maxRequestBody?: number;
}
/** What one pass changed. */
@@ -156,10 +166,22 @@ export function routesFrom(document: unknown, machine: string): { routes: Route[
skipped.push(`${from} asked for route ${name} and gave no usable port`);
continue;
}
// A limit it cannot honour is a route it does not write — skipped and named, like a port that
// is not one. Written without the limit instead, the predecessor would carry exactly what the
// module said not to carry, and this module would report success.
const asked_limit = entry.values?.["max-request-body"];
const limit = asBodyLimit(asked_limit);
if (limit === null) {
skipped.push(
`${from} asked for route ${name} with a max-request-body of ${JSON.stringify(asked_limit)}, ` +
`which is not a whole positive number of bytes`,
);
continue;
}
// Where the mesh says that machine is. Empty means this one, and this one is reached from
// inside the predecessor's container by the machine's own name, not by loopback.
const at = typeof entry.at === "string" && entry.at.trim() !== "" ? entry.at.trim() : machine;
routes.push({ name, from, target: `http://${at}:${port}` });
routes.push({ name, from, target: `http://${at}:${port}`, ...(limit === undefined ? {} : { maxRequestBody: limit }) });
}
return { routes, skipped };
}
@@ -183,6 +205,10 @@ export function routerNameFor(name: string): string {
*/
export function routeFile(route: Route, settings: Settings): string {
const id = routerNameFor(route.name);
// The body limit is a middleware in the predecessor's vocabulary — its `buffering`, with the
// one field the predecessor's own registry route set — named after the router so the two halves
// cannot drift, and written only when the contribution asked for it.
const limited = route.maxRequestBody !== undefined;
return [
marker,
`# ${route.from} contributed this route. It is removed when that contribution goes.`,
@@ -192,10 +218,19 @@ export function routeFile(route: Route, settings: Settings): string {
` entryPoints: [${settings.entrypoint}]`,
` rule: Host(\`${route.name}\`)`,
` service: ${id}`,
...(limited ? [` middlewares: [${id}-body]`] : []),
" tls:",
` certResolver: ${settings.resolver}`,
" domains:",
` - main: ${route.name}`,
...(limited
? [
" middlewares:",
` ${id}-body:`,
" buffering:",
` maxRequestBodyBytes: ${route.maxRequestBody}`,
]
: []),
" services:",
` ${id}:`,
" loadBalancer:",
@@ -279,6 +314,22 @@ export async function reconcile(routes: Route[], settings: Settings): Promise<Pa
return pass;
}
/**
* A contribution's `max-request-body`: `undefined` when it said nothing, the number of bytes when
* it is a whole positive number, and `null` when it is anything else — the same rule the
* controller's catalogue applies when it parses the manifest, so a limit that reaches here has
* already passed it once, and one that fails it was laid over by a setting.
*/
function asBodyLimit(value: unknown): number | undefined | null {
if (value === undefined) {
return undefined;
}
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
return null;
}
return value;
}
/** What JSON makes of a port, which is a float even where it was written 8080. */
function asPort(value: unknown): number | undefined {
const port = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN;
modules/route-adapter/test/adapter.test.ts
+65 -2
View File
@@ -22,7 +22,9 @@ async function predecessor(already: Record<string, string> = {}): Promise<Settin
}
/** The contributions file the mesh writes, in the shape the mesh's own proxy also reads. */
function contributed(...given: { from: string; node?: string; at?: string; name: string; port: number }[]) {
function contributed(
...given: { from: string; node?: string; at?: string; name: string; port: number; limit?: unknown }[]
) {
return {
contributions: 1,
requirement: "route",
@@ -30,7 +32,7 @@ function contributed(...given: { from: string; node?: string; at?: string; name:
from: g.from,
node: g.node ?? "control-node",
at: g.at ?? "",
values: { name: g.name, port: g.port },
values: { name: g.name, port: g.port, ...(g.limit === undefined ? {} : { "max-request-body": g.limit }) },
})),
};
}
@@ -211,3 +213,64 @@ test("it refuses when the predecessor's directory is not there, and says why", a
const settings = { ...defaults, dynamic: join(await mkdtemp(join(tmpdir(), "route-adapter-")), "absent") };
await assert.rejects(reconcile([], settings), /is not there.*`dynamic` setting.*mounts it/s);
});
// **The registry's hand-over** (novox/hq ADR 0082, ADR 0104). A registry takes image layers in
// single requests of gigabytes, and the predecessor served its public name with a twenty-gigabyte
// `buffering` middleware. The contribution carries that limit as `max-request-body`; the adapter
// writes it as the middleware the predecessor already understands, named after the router, and
// writes nothing of the kind for a route that did not ask.
test("a body limit is written as the predecessor's buffering middleware", async () => {
const settings = await predecessor();
const changed = await pass(settings, contributed(
{ from: "distribution-gate", name: "registry-api.example", port: 5001, limit: 21474836480 },
{ from: "gitea", name: "git.example", port: 2999 },
));
assert.deepEqual(changed.written, ["mesh-git.example.yml", "mesh-registry-api.example.yml"]);
assert.equal(await readFile(join(settings.dynamic, "mesh-registry-api.example.yml"), "utf8"), [
marker,
"# distribution-gate contributed this route. It is removed when that contribution goes.",
"http:",
" routers:",
" mesh-registry-api-example:",
" entryPoints: [websecure]",
" rule: Host(`registry-api.example`)",
" service: mesh-registry-api-example",
" middlewares: [mesh-registry-api-example-body]",
" tls:",
" certResolver: le",
" domains:",
" - main: registry-api.example",
" middlewares:",
" mesh-registry-api-example-body:",
" buffering:",
" maxRequestBodyBytes: 21474836480",
" services:",
" mesh-registry-api-example:",
" loadBalancer:",
" servers:",
" - url: http://host.docker.internal:5001",
"",
].join("\n"));
// The route that asked for nothing carries no middleware — the predecessor's default stands.
const plain = await readFile(join(settings.dynamic, "mesh-git.example.yml"), "utf8");
assert.doesNotMatch(plain, /middlewares|buffering/);
});
// A limit it cannot honour is a route it does not write. Written without it, the predecessor would
// carry exactly what the module said not to carry, and this module would report success.
test("a body limit that is not a number of bytes is skipped and named", () => {
const machine = defaults.machine;
for (const limit of ["20g", 0, -1, 1.5, true, null]) {
const { routes, skipped } = routesFrom(
contributed({ from: "gate", name: "registry-api.example", port: 5001, limit }), machine);
assert.deepEqual(routes, [], `a limit of ${JSON.stringify(limit)} was served`);
assert.equal(skipped.length, 1);
assert.match(skipped[0]!, /max-request-body/);
}
// And a limit the controller would accept is carried, as a number.
const { routes } = routesFrom(
contributed({ from: "gate", name: "registry-api.example", port: 5001, limit: 1024 }), machine);
assert.equal(routes[0]?.maxRequestBody, 1024);
});