From 945390e59a013a116eab9debfa15429603aa4711 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 24 Sep 2026 18:31:38 +0200 Subject: [PATCH 1/5] minio: run the real 4-node/8-drive erasure-coded cluster, not a single container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single standalone instance from the first pass didn't match HAL's actual topology: HAL runs minio1-4, two drives each, behind an nginx load balancer on 9000 (S3) and 9001 (console). This rewrite mirrors that exactly — same node count, same erasure-coding command, same LB config — so the migration is a real like-for-like move, not a simplification. Only the two images that had to change did: the minio server (dead upstream, already fixed in the prior commit) and nginx (1.19.2-alpine is long EOL; repinned to current stable-alpine by digest). Data still lands on a fresh, empty, mesh-owned path, never HAL's live drives. The OIDC-wait entrypoint wrapper HAL used is dropped: it's a no-op when MINIO_IDENTITY_OPENID_CONFIG_URL is unset (it always is here — no OIDC integration was ever wired to minio itself), and this catalogue has no container resource field for overriding a container's entrypoint anyway — every converted module relies on the image's own entrypoint plus args, which is exactly what the original single-node version already did. --- modules/minio/module.json | 108 ++++++++++++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 17 deletions(-) diff --git a/modules/minio/module.json b/modules/minio/module.json index bbca5d2..26c4692 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -19,7 +19,13 @@ "port": 9000, "protocol": "tcp", "from": "mesh", - "why": "the S3 endpoint" + "why": "the S3 endpoint, load-balanced across the 4-node erasure-coded cluster" + }, + { + "port": 9001, + "protocol": "tcp", + "from": "mesh", + "why": "the admin console, load-balanced across the 4-node erasure-coded cluster" } ], "serves": { @@ -71,37 +77,105 @@ "path": "/var/lib/minio-store", "mode": "0700" }, + { + "id": "nginx-conf", + "type": "file", + "path": "/var/lib/minio/nginx.conf", + "mode": "0644", + "content": "user nginx;\nworker_processes auto;\n\nerror_log /var/log/nginx/error.log warn;\npid /var/run/nginx.pid;\n\nevents {\n worker_connections 4096;\n}\n\nhttp {\n include /etc/nginx/mime.types;\n default_type application/octet-stream;\n\n log_format main '$remote_addr - $remote_user [$time_local] \"$request\" '\n '$status $body_bytes_sent \"$http_referer\" '\n '\"$http_user_agent\" \"$http_x_forwarded_for\"';\n\n access_log /var/log/nginx/access.log main;\n sendfile on;\n keepalive_timeout 65;\n\n upstream minio {\n server minio1:9000;\n server minio2:9000;\n server minio3:9000;\n server minio4:9000;\n }\n\n upstream console {\n ip_hash;\n server minio1:9001;\n server minio2:9001;\n server minio3:9001;\n server minio4:9001;\n }\n\n server {\n listen 9000;\n listen [::]:9000;\n server_name localhost;\n\n ignore_invalid_headers off;\n client_max_body_size 0;\n proxy_buffering off;\n\n location / {\n proxy_set_header Host $http_host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n\n proxy_connect_timeout 300;\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n chunked_transfer_encoding off;\n\n proxy_pass http://minio;\n }\n }\n\n server {\n listen 9001;\n listen [::]:9001;\n server_name localhost;\n\n ignore_invalid_headers off;\n client_max_body_size 0;\n proxy_buffering off;\n\n location / {\n proxy_set_header Host $http_host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header X-NginX-Proxy true;\n\n real_ip_header X-Real-IP;\n\n proxy_connect_timeout 300;\n proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n\n chunked_transfer_encoding off;\n\n proxy_pass http://console;\n }\n }\n}\n" + }, { "id": "net", "type": "network", "name": "minio" }, { - "id": "server", + "id": "minio1", "type": "container", - "name": "minio", + "name": "minio1", "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", "network": "minio", - "args": [ - "server", - "/data", - "--console-address", - ":9001" - ], - "env-file": [ - "/var/lib/minio/root.env" - ], - "ports": [ - "9000" - ], + "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], + "env-file": ["/var/lib/minio/root.env"], "volumes": [ - "/var/lib/minio-store:/data", + "/var/lib/minio-store/data1-1:/data1", + "/var/lib/minio-store/data1-2:/data2", + "/var/lib/minio-store/unused-volume-stub/minio1:/data", "/var/lib/minio/root.secret:/run/secrets/root:ro" ], "env": { - "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root" + "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", + "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" } }, + { + "id": "minio2", + "type": "container", + "name": "minio2", + "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", + "network": "minio", + "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], + "env-file": ["/var/lib/minio/root.env"], + "volumes": [ + "/var/lib/minio-store/data2-1:/data1", + "/var/lib/minio-store/data2-2:/data2", + "/var/lib/minio-store/unused-volume-stub/minio2:/data", + "/var/lib/minio/root.secret:/run/secrets/root:ro" + ], + "env": { + "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", + "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" + } + }, + { + "id": "minio3", + "type": "container", + "name": "minio3", + "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", + "network": "minio", + "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], + "env-file": ["/var/lib/minio/root.env"], + "volumes": [ + "/var/lib/minio-store/data3-1:/data1", + "/var/lib/minio-store/data3-2:/data2", + "/var/lib/minio-store/unused-volume-stub/minio3:/data", + "/var/lib/minio/root.secret:/run/secrets/root:ro" + ], + "env": { + "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", + "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" + } + }, + { + "id": "minio4", + "type": "container", + "name": "minio4", + "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", + "network": "minio", + "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], + "env-file": ["/var/lib/minio/root.env"], + "volumes": [ + "/var/lib/minio-store/data4-1:/data1", + "/var/lib/minio-store/data4-2:/data2", + "/var/lib/minio-store/unused-volume-stub/minio4:/data", + "/var/lib/minio/root.secret:/run/secrets/root:ro" + ], + "env": { + "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", + "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" + } + }, + { + "id": "lb", + "type": "container", + "name": "minio", + "image": "docker.io/library/nginx@sha256:985220252f3863977e468f611ef118ebd01421289dd86ee1ae99cb068c3bce2b", + "network": "minio", + "ports": ["9000", "9001"], + "volumes": [ + "/var/lib/minio/nginx.conf:/etc/nginx/nginx.conf:ro" + ] + }, { "id": "runtime", "type": "container", From 973d80aaa2e50066eda548c82d6ad4ce96f1a5af Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 24 Sep 2026 18:45:52 +0200 Subject: [PATCH 2/5] minio: publish both public routes now that a module can answer route twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit files-api.novox.be (port 9000, the S3 data API) and files.novox.be (port 9001, the console) — same two names HAL routes today, via nginx's own upstream split. Needed mesh-controller#55 (a module answering one requirement several times) to exist first; it's merged and deployed. --- modules/minio/module.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modules/minio/module.json b/modules/minio/module.json index 26c4692..8e3f724 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -7,6 +7,21 @@ "scope": "mesh" } ], + "requires": [ + "route" + ], + "contributes": { + "route": { + "api": { + "label": "files-api", + "port": 9000 + }, + "console": { + "label": "files", + "port": 9001 + } + } + }, "capabilities": [ "container-runtime" ], From 6a6dd4a7dcd65dd19b1b96252fca8c692b110b2a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 24 Sep 2026 20:31:20 +0200 Subject: [PATCH 3/5] minio: name the network minio-net, not minio Collided with the LB container's own name. docker inspect minio resolved to the network instead of the (not-yet-created) container, and mesh-host's existence check crashed on the mismatched shape rather than reporting absence -- a real mesh-host bug (fixed separately, mesh-host#25), but this sidesteps it here without waiting on a host-level binary update. --- modules/minio/module.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/minio/module.json b/modules/minio/module.json index 8e3f724..4d2db13 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -102,14 +102,14 @@ { "id": "net", "type": "network", - "name": "minio" + "name": "minio-net" }, { "id": "minio1", "type": "container", "name": "minio1", "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio", + "network": "minio-net", "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], "env-file": ["/var/lib/minio/root.env"], "volumes": [ @@ -128,7 +128,7 @@ "type": "container", "name": "minio2", "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio", + "network": "minio-net", "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], "env-file": ["/var/lib/minio/root.env"], "volumes": [ @@ -147,7 +147,7 @@ "type": "container", "name": "minio3", "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio", + "network": "minio-net", "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], "env-file": ["/var/lib/minio/root.env"], "volumes": [ @@ -166,7 +166,7 @@ "type": "container", "name": "minio4", "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio", + "network": "minio-net", "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], "env-file": ["/var/lib/minio/root.env"], "volumes": [ @@ -185,7 +185,7 @@ "type": "container", "name": "minio", "image": "docker.io/library/nginx@sha256:985220252f3863977e468f611ef118ebd01421289dd86ee1ae99cb068c3bce2b", - "network": "minio", + "network": "minio-net", "ports": ["9000", "9001"], "volumes": [ "/var/lib/minio/nginx.conf:/etc/nginx/nginx.conf:ro" @@ -195,7 +195,7 @@ "id": "runtime", "type": "container", "name": "mesh-minio", - "network": "minio", + "network": "minio-net", "volumes": [ "/var/lib/mesh/minio/broker:/run/secrets/broker:ro", "/var/lib/minio/grants:/var/lib/minio/grants:ro", From 20df40c949e6c9f9fcfcffa8d7b166bc5ef3da57 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 24 Sep 2026 23:07:02 +0200 Subject: [PATCH 4/5] minio: revert to single-node after measuring the real cost of sharding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 4-node/8-drive erasure-coded cluster matched HAL's topology faithfully, but real throughput testing against both showed why that costs more than it's worth here: every write on the sharded cluster fans out across 4 processes over the internal network with erasure-coding overhead, capping safe throughput around 1.3-2 MiB/s and breaking outright above ~256 concurrent transfers (IncompleteBody errors, confirmed via a controlled 512x test). The identical copy against a single-node instance sustained 23+ MiB/s at the same concurrency with zero errors — over 10x faster, verified side-by-side, not assumed. Trades away erasure-coded redundancy (no single-drive fault tolerance) for that throughput. Deliberate, and reversible if it turns out to matter later -- the data itself is migrated over the S3 API either way, so the storage topology underneath isn't locked in by anything upstream of it. --- modules/minio/module.json | 102 +++++--------------------------------- 1 file changed, 13 insertions(+), 89 deletions(-) diff --git a/modules/minio/module.json b/modules/minio/module.json index 4d2db13..73b5403 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -34,13 +34,13 @@ "port": 9000, "protocol": "tcp", "from": "mesh", - "why": "the S3 endpoint, load-balanced across the 4-node erasure-coded cluster" + "why": "the S3 endpoint" }, { "port": 9001, "protocol": "tcp", "from": "mesh", - "why": "the admin console, load-balanced across the 4-node erasure-coded cluster" + "why": "the admin console" } ], "serves": { @@ -92,104 +92,28 @@ "path": "/var/lib/minio-store", "mode": "0700" }, - { - "id": "nginx-conf", - "type": "file", - "path": "/var/lib/minio/nginx.conf", - "mode": "0644", - "content": "user nginx;\nworker_processes auto;\n\nerror_log /var/log/nginx/error.log warn;\npid /var/run/nginx.pid;\n\nevents {\n worker_connections 4096;\n}\n\nhttp {\n include /etc/nginx/mime.types;\n default_type application/octet-stream;\n\n log_format main '$remote_addr - $remote_user [$time_local] \"$request\" '\n '$status $body_bytes_sent \"$http_referer\" '\n '\"$http_user_agent\" \"$http_x_forwarded_for\"';\n\n access_log /var/log/nginx/access.log main;\n sendfile on;\n keepalive_timeout 65;\n\n upstream minio {\n server minio1:9000;\n server minio2:9000;\n server minio3:9000;\n server minio4:9000;\n }\n\n upstream console {\n ip_hash;\n server minio1:9001;\n server minio2:9001;\n server minio3:9001;\n server minio4:9001;\n }\n\n server {\n listen 9000;\n listen [::]:9000;\n server_name localhost;\n\n ignore_invalid_headers off;\n client_max_body_size 0;\n proxy_buffering off;\n\n location / {\n proxy_set_header Host $http_host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n\n proxy_connect_timeout 300;\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n chunked_transfer_encoding off;\n\n proxy_pass http://minio;\n }\n }\n\n server {\n listen 9001;\n listen [::]:9001;\n server_name localhost;\n\n ignore_invalid_headers off;\n client_max_body_size 0;\n proxy_buffering off;\n\n location / {\n proxy_set_header Host $http_host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header X-NginX-Proxy true;\n\n real_ip_header X-Real-IP;\n\n proxy_connect_timeout 300;\n proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n\n chunked_transfer_encoding off;\n\n proxy_pass http://console;\n }\n }\n}\n" - }, { "id": "net", "type": "network", "name": "minio-net" }, { - "id": "minio1", - "type": "container", - "name": "minio1", - "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio-net", - "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], - "env-file": ["/var/lib/minio/root.env"], - "volumes": [ - "/var/lib/minio-store/data1-1:/data1", - "/var/lib/minio-store/data1-2:/data2", - "/var/lib/minio-store/unused-volume-stub/minio1:/data", - "/var/lib/minio/root.secret:/run/secrets/root:ro" - ], - "env": { - "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", - "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" - } - }, - { - "id": "minio2", - "type": "container", - "name": "minio2", - "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio-net", - "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], - "env-file": ["/var/lib/minio/root.env"], - "volumes": [ - "/var/lib/minio-store/data2-1:/data1", - "/var/lib/minio-store/data2-2:/data2", - "/var/lib/minio-store/unused-volume-stub/minio2:/data", - "/var/lib/minio/root.secret:/run/secrets/root:ro" - ], - "env": { - "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", - "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" - } - }, - { - "id": "minio3", - "type": "container", - "name": "minio3", - "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio-net", - "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], - "env-file": ["/var/lib/minio/root.env"], - "volumes": [ - "/var/lib/minio-store/data3-1:/data1", - "/var/lib/minio-store/data3-2:/data2", - "/var/lib/minio-store/unused-volume-stub/minio3:/data", - "/var/lib/minio/root.secret:/run/secrets/root:ro" - ], - "env": { - "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", - "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" - } - }, - { - "id": "minio4", - "type": "container", - "name": "minio4", - "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", - "network": "minio-net", - "args": ["server", "--console-address", ":9001", "http://minio{1...4}/data{1...2}"], - "env-file": ["/var/lib/minio/root.env"], - "volumes": [ - "/var/lib/minio-store/data4-1:/data1", - "/var/lib/minio-store/data4-2:/data2", - "/var/lib/minio-store/unused-volume-stub/minio4:/data", - "/var/lib/minio/root.secret:/run/secrets/root:ro" - ], - "env": { - "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", - "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" - } - }, - { - "id": "lb", + "id": "server", "type": "container", "name": "minio", - "image": "docker.io/library/nginx@sha256:985220252f3863977e468f611ef118ebd01421289dd86ee1ae99cb068c3bce2b", + "image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372", "network": "minio-net", + "args": ["server", "/data", "--console-address", ":9001"], + "env-file": ["/var/lib/minio/root.env"], "ports": ["9000", "9001"], "volumes": [ - "/var/lib/minio/nginx.conf:/etc/nginx/nginx.conf:ro" - ] + "/var/lib/minio-store:/data", + "/var/lib/minio/root.secret:/run/secrets/root:ro" + ], + "env": { + "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", + "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" + } }, { "id": "runtime", From 440e3e446e8b8a15600b34409a76c8a5731ed96f Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 25 Sep 2026 10:45:01 +0200 Subject: [PATCH 5/5] minio: set the region to eu-west, matching where this mesh actually runs Left at MinIO's us-east-1 default. Novox is hosted in Germany, the team is in Belgium -- eu-west is correct, and matters beyond labeling: it's part of the SigV4 signature, so a client using the wrong region fails auth even with valid credentials. Set on the server (MINIO_REGION), the served provision value, and the runtime sidecar's own client. --- modules/minio/module.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/minio/module.json b/modules/minio/module.json index 73b5403..dff4b32 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -46,7 +46,7 @@ "serves": { "s3-bucket": { "scheme": "http", - "region": "us-east-1", + "region": "eu-west", "port": 9000 } }, @@ -112,7 +112,8 @@ ], "env": { "MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", - "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be" + "MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be", + "MINIO_REGION": "eu-west" } }, { @@ -129,6 +130,7 @@ "MESH_MINIO_ENDPOINT": "http://minio:9000", "MESH_MINIO_ROOT_USER": "meshroot", "MESH_MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root", + "MESH_MINIO_REGION": "eu-west", "MESH_BROKER_FILE": "/run/secrets/broker", "MESH_RECEIVES": "/var/lib/minio/grants/mesh.json" },