From 661114370f31703df2b377dd15ebff03a8329d56 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 2 Oct 2026 21:24:43 +0200 Subject: [PATCH 1/4] minio declares the bucket it derives; its consumers stop transcribing it (hq ADR 0188) serves.s3-bucket.bucket is ${consumer:as:dns}; the provisioner uses what it is given. nextcloud, invoicing and photos ask for ${bound:s3-bucket:bucket} instead of naming mesh-novox-* literals, which also named this node. bucketFor and the long-dead accessKeyFor are gone. --- modules/invoicing/module.json | 2 +- modules/minio/client.ts | 20 ++++------------ modules/minio/module.json | 3 ++- modules/minio/package.json | 2 +- modules/minio/provisioner/index.ts | 38 ++++++++++++++++++++++++------ modules/nextcloud/module.json | 2 +- modules/photos/module.json | 2 +- 7 files changed, 42 insertions(+), 27 deletions(-) diff --git a/modules/invoicing/module.json b/modules/invoicing/module.json index 31499bf..9f8dfcf 100644 --- a/modules/invoicing/module.json +++ b/modules/invoicing/module.json @@ -68,7 +68,7 @@ "type": "file", "path": "${dir:state}/api.env", "mode": "0600", - "content": "NODE_ENV=production\nPORT=9000\nMONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=${bound:mongodb-database:as}\nMONGO_DB=${bound:mongodb-database:as}\nMINIO_BUCKET=mesh-novox-invoice\nMINIO_ENDPOINT=${bound:s3-bucket:at}\nMINIO_PORT=${bound:s3-bucket:port}\nMINIO_ACCESSKEY=${bound:s3-bucket:as}\nMINIO_SECRET=${secret:s3-bucket}\n" + "content": "NODE_ENV=production\nPORT=9000\nMONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=${bound:mongodb-database:as}\nMONGO_DB=${bound:mongodb-database:as}\nMINIO_BUCKET=${bound:s3-bucket:bucket}\nMINIO_ENDPOINT=${bound:s3-bucket:at}\nMINIO_PORT=${bound:s3-bucket:port}\nMINIO_ACCESSKEY=${bound:s3-bucket:as}\nMINIO_SECRET=${secret:s3-bucket}\n" }, { "id": "net", diff --git a/modules/minio/client.ts b/modules/minio/client.ts index beb751f..999f753 100644 --- a/modules/minio/client.ts +++ b/modules/minio/client.ts @@ -303,21 +303,11 @@ export class MinioClient { // --- module-scoped helpers ------------------------------------------------- -/** A deterministic 20-char access key id from a consumer name, so removal needs no stored state: - * the provisioner recomputes the same id at teardown that it minted at creation. */ -export function accessKeyFor(consumer: string): string { - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - const digest = createHash("sha256").update(consumer).digest(); - let out = ""; - for (let i = 0; i < 20; i++) out += chars[digest[i] % chars.length]; - return out; -} - -/** A DNS-safe bucket name derived from a consumer — the removable identity of its storage. */ -export function bucketFor(consumer: string): string { - const name = consumer.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63); - return name.length >= 3 ? name : `mesh-${name}`; -} +// **Neither the access key nor the bucket is derived here any more.** `accessKeyFor` minted an id +// of its own until the mesh took that over (ADR 0048: the login is the mesh's, handed to both +// ends), and `bucketFor` derived the bucket until the mesh took that over too (ADR 0188: the rule +// is a line of this module's manifest, filled per consumer and delivered to both ends). Both +// survived with no callers, which is the state a rule comes back from; they are gone. function bucketPolicy(bucket: string): string { return JSON.stringify({ diff --git a/modules/minio/module.json b/modules/minio/module.json index 1d1f045..6a0bc85 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -49,7 +49,8 @@ "s3-bucket": { "scheme": "http", "region": "eu-west", - "port": 9000 + "port": 9000, + "bucket": "${consumer:as:dns}" } }, "receives": { diff --git a/modules/minio/package.json b/modules/minio/package.json index 7441fc6..175a93d 100644 --- a/modules/minio/package.json +++ b/modules/minio/package.json @@ -5,7 +5,7 @@ "type": "module", "private": true, "dependencies": { - "@novox/mesh-sdk": "^0.1.1" + "@novox/mesh-sdk": "^0.1.2" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/modules/minio/provisioner/index.ts b/modules/minio/provisioner/index.ts index 4c34201..dfaf2d2 100644 --- a/modules/minio/provisioner/index.ts +++ b/modules/minio/provisioner/index.ts @@ -10,18 +10,24 @@ // **The access key and its secret are the mesh's, not the provisioner's (ADR 0048).** The mesh // derives the login (the access-key id) and hands it to both ends, and mints the secret key. minio // creates the service account under exactly that access key with exactly that secret — a credential -// the provisioner invented is one the consumer could never present. The bucket is derived from the -// login, so teardown recomputes it with nothing to persist. +// the provisioner invented is one the consumer could never present. +// +// **The bucket name is the mesh's too (ADR 0188).** It used to be computed here, from the login, +// and every consumer transcribed the same rule into its own definition by hand — two copies of +// one rule with nothing comparing them, and one of three was wrong for months. Now the rule is a +// line of this module's manifest (`serves.s3-bucket.bucket: ${consumer:as:dns}`), the mesh fills +// it per consumer, and the same filled value reaches this provisioner and the consumer's own +// configuration. There is no second computation to disagree with. import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner"; import { emit } from "@novox/mesh-sdk/events"; -import { MinioClient, bucketFor } from "../client.js"; +import { MinioClient } from "../client.js"; const minio = MinioClient.fromEnv(); runProvisioner("s3-bucket", { async create(p: Provision): Promise { - const bucket = bucketFor(p.as); + const bucket = bucketNamed(p.derived); const accessKeyId = p.as; if (!(await minio.bucketExists(bucket))) await minio.createBucket(bucket); @@ -38,8 +44,8 @@ runProvisioner("s3-bucket", { }); }, - async remove(p: { as: string }): Promise { - const bucket = bucketFor(p.as); + async remove(p: { as: string; derived: Readonly> }): Promise { + const bucket = bucketNamed(p.derived); // Revoking the key is what cuts the consumer's access. The bucket is emptied-then-dropped only if // empty; a bucket that still holds objects is left for an operator rather than erroring on every @@ -57,10 +63,28 @@ runProvisioner("s3-bucket", { // Asked every minute by the harness: whether the backend still holds this consumer exactly as // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120). async holds(p: Provision): Promise { - return minio.canReachAs(bucketFor(p.as), p.as, p.password); + return minio.canReachAs(bucketNamed(p.derived), p.as, p.password); }, }); +/** The bucket the mesh derived for this consumer. + * + * Absent means this module is running against a control plane that does not fill `${consumer:…}` + * yet, or a manifest whose `serves` block lost the line. Both are the same mistake from here — + * nobody said which bucket — and both are said rather than guessed: a provisioner that fell back + * to deriving one would restore the second rule and hide the fault behind a bucket that happens + * to be right. */ +function bucketNamed(derived: Readonly>): string { + const bucket = derived.bucket; + if (typeof bucket !== "string" || bucket === "") { + throw new Error( + "the mesh did not say which bucket this consumer gets: minio's manifest must serve " + + "`bucket` under s3-bucket (novox/hq ADR 0188)", + ); + } + return bucket; +} + /** Emit best-effort: a broker hiccup is logged and dropped, never allowed to throw back and fail a * bucket that was made. */ async function announce(type: string, body: unknown): Promise { diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index fbcadd2..39fac38 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -62,7 +62,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "POSTGRES_HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nPOSTGRES_DB=${bound:postgres-database:as}\nPOSTGRES_USER=${bound:postgres-database:as}\nPOSTGRES_PASSWORD=${secret:postgres-database}\nNEXTCLOUD_ADMIN_USER=mesh-admin\nNEXTCLOUD_ADMIN_PASSWORD=${secret:admin}\nOBJECTSTORE_S3_HOST=${bound:s3-bucket:at}\nOBJECTSTORE_S3_PORT=${bound:s3-bucket:port}\nOBJECTSTORE_S3_BUCKET=mesh-novox-ncloud\nOBJECTSTORE_S3_KEY=${bound:s3-bucket:as}\nOBJECTSTORE_S3_SECRET=${secret:s3-bucket}\nOBJECTSTORE_S3_SSL=false\nOBJECTSTORE_S3_USEPATH_STYLE=true\nOBJECTSTORE_S3_REGION=${bound:s3-bucket:region}\n" + "content": "POSTGRES_HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nPOSTGRES_DB=${bound:postgres-database:as}\nPOSTGRES_USER=${bound:postgres-database:as}\nPOSTGRES_PASSWORD=${secret:postgres-database}\nNEXTCLOUD_ADMIN_USER=mesh-admin\nNEXTCLOUD_ADMIN_PASSWORD=${secret:admin}\nOBJECTSTORE_S3_HOST=${bound:s3-bucket:at}\nOBJECTSTORE_S3_PORT=${bound:s3-bucket:port}\nOBJECTSTORE_S3_BUCKET=${bound:s3-bucket:bucket}\nOBJECTSTORE_S3_KEY=${bound:s3-bucket:as}\nOBJECTSTORE_S3_SECRET=${secret:s3-bucket}\nOBJECTSTORE_S3_SSL=false\nOBJECTSTORE_S3_USEPATH_STYLE=true\nOBJECTSTORE_S3_REGION=${bound:s3-bucket:region}\n" }, { "id": "html", diff --git a/modules/photos/module.json b/modules/photos/module.json index 8cc9c74..7ea1556 100644 --- a/modules/photos/module.json +++ b/modules/photos/module.json @@ -58,7 +58,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "NODE_ENV=production\nPORT=9000\nMONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=admin\nMONGO_DB=${bound:mongodb-database:as}\nMINIO_ENDPOINT=${bound:s3-bucket:at}\nMINIO_PORT=${bound:s3-bucket:port}\nMINIO_BUCKET=mesh-novox-photos\nMINIO_ACCESSKEY=${bound:s3-bucket:as}\nMINIO_SECRET=${secret:s3-bucket}\nMINIO_USE_SSL=false\n" + "content": "NODE_ENV=production\nPORT=9000\nMONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=admin\nMONGO_DB=${bound:mongodb-database:as}\nMINIO_ENDPOINT=${bound:s3-bucket:at}\nMINIO_PORT=${bound:s3-bucket:port}\nMINIO_BUCKET=${bound:s3-bucket:bucket}\nMINIO_ACCESSKEY=${bound:s3-bucket:as}\nMINIO_SECRET=${secret:s3-bucket}\nMINIO_USE_SSL=false\n" }, { "id": "net", From 723e676b75097a6e7c16f83525abdd5977dbea2a Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 2 Oct 2026 21:49:03 +0200 Subject: [PATCH 2/4] The store enables deletion and collects nightly (hq ADR 0189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REGISTRY_STORAGE_DELETE_ENABLED on the server — the door already accepts a push — and a scheduled step running the registry's own collector over the volume at 03:30 with the server held still. Plain garbage-collect: what the mesh keeps is still a manifest, so --delete-untagged is not needed and would delete images machines are running. --- modules/distribution/module.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 1da7c8f..3a7c85f 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -59,6 +59,26 @@ ], "volumes": [ "/var/lib/mesh-registry:/var/lib/registry" + ], + "env": { + "REGISTRY_STORAGE_DELETE_ENABLED": "true" + } + }, + { + "id": "collect", + "type": "container", + "name": "mesh-registry-collect", + "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "volumes": [ + "/var/lib/mesh-registry:/var/lib/registry" + ], + "args": [ + "garbage-collect", + "/etc/docker/registry/config.yml" + ], + "schedule": "30 3 * * *", + "while-stopped": [ + "store" ] } ] From 159ed53103505d41b9f4e5e07395e16dc862a937 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 4 Oct 2026 02:45:13 +0200 Subject: [PATCH 3/4] Rebased onto main: ADR 0188 renumbered to 0201, and minio takes the sdk at 0.1.7 The bundles refactor took ADR 0188 on main, so minio's comments cite 0201. The sdk is 0.1.7 after the same rebase, and minio needs the `derived` field it carries. --- modules/minio/client.ts | 2 +- modules/minio/package.json | 2 +- modules/minio/provisioner/index.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/minio/client.ts b/modules/minio/client.ts index 999f753..f40edf4 100644 --- a/modules/minio/client.ts +++ b/modules/minio/client.ts @@ -305,7 +305,7 @@ export class MinioClient { // **Neither the access key nor the bucket is derived here any more.** `accessKeyFor` minted an id // of its own until the mesh took that over (ADR 0048: the login is the mesh's, handed to both -// ends), and `bucketFor` derived the bucket until the mesh took that over too (ADR 0188: the rule +// ends), and `bucketFor` derived the bucket until the mesh took that over too (ADR 0201: the rule // is a line of this module's manifest, filled per consumer and delivered to both ends). Both // survived with no callers, which is the state a rule comes back from; they are gone. diff --git a/modules/minio/package.json b/modules/minio/package.json index 175a93d..892b097 100644 --- a/modules/minio/package.json +++ b/modules/minio/package.json @@ -5,7 +5,7 @@ "type": "module", "private": true, "dependencies": { - "@novox/mesh-sdk": "^0.1.2" + "@novox/mesh-sdk": "^0.1.7" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/modules/minio/provisioner/index.ts b/modules/minio/provisioner/index.ts index dfaf2d2..fca498d 100644 --- a/modules/minio/provisioner/index.ts +++ b/modules/minio/provisioner/index.ts @@ -12,7 +12,7 @@ // creates the service account under exactly that access key with exactly that secret — a credential // the provisioner invented is one the consumer could never present. // -// **The bucket name is the mesh's too (ADR 0188).** It used to be computed here, from the login, +// **The bucket name is the mesh's too (ADR 0201).** It used to be computed here, from the login, // and every consumer transcribed the same rule into its own definition by hand — two copies of // one rule with nothing comparing them, and one of three was wrong for months. Now the rule is a // line of this module's manifest (`serves.s3-bucket.bucket: ${consumer:as:dns}`), the mesh fills @@ -79,7 +79,7 @@ function bucketNamed(derived: Readonly>): string { if (typeof bucket !== "string" || bucket === "") { throw new Error( "the mesh did not say which bucket this consumer gets: minio's manifest must serve " + - "`bucket` under s3-bucket (novox/hq ADR 0188)", + "`bucket` under s3-bucket (novox/hq ADR 0201)", ); } return bucket; From 525c6390419dfb04b35c20ec99da8ca0c1b0eea7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 4 Oct 2026 04:26:38 +0200 Subject: [PATCH 4/4] The store does not collect until every controller composes the window (hq ADR 0189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh-controller#259 fixes while-stopped to name the container as the machine knows it — `distribution.store`, not `store`. Until that controller is the one composing, novox refuses its whole declaration and takes nothing at all. The step comes out; deletion stays on, already applied and harmless on its own. A collect step without its window would be worse than none: garbage collection against a live registry can sweep a blob a build is pushing. Put back once the fixed controller is deployed and stays. --- modules/distribution/module.json | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 3a7c85f..5f2c1b7 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -63,23 +63,6 @@ "env": { "REGISTRY_STORAGE_DELETE_ENABLED": "true" } - }, - { - "id": "collect", - "type": "container", - "name": "mesh-registry-collect", - "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", - "volumes": [ - "/var/lib/mesh-registry:/var/lib/registry" - ], - "args": [ - "garbage-collect", - "/etc/docker/registry/config.yml" - ], - "schedule": "30 3 * * *", - "while-stopped": [ - "store" - ] } ] }