jschoubben
20df40c949
The 4-node/8-drive erasure-coded cluster matched HAL's topology faithfully, but real throughput testing against both showed why that costs more than it's worth here: every write on the sharded cluster fans out across 4 processes over the internal network with erasure-coding overhead, capping safe throughput around 1.3-2 MiB/s and breaking outright above ~256 concurrent transfers (IncompleteBody errors, confirmed via a controlled 512x test). The identical copy against a single-node instance sustained 23+ MiB/s at the same concurrency with zero errors — over 10x faster, verified side-by-side, not assumed. Trades away erasure-coded redundancy (no single-drive fault tolerance) for that throughput. Deliberate, and reversible if it turns out to matter later -- the data itself is migrated over the S3 API either way, so the storage topology underneath isn't locked in by anything upstream of it.
160 lines
3.4 KiB
JSON
160 lines
3.4 KiB
JSON
{
|
|
"module": "minio",
|
|
"version": "1",
|
|
"provides": [
|
|
{
|
|
"name": "s3-bucket",
|
|
"scope": "mesh"
|
|
}
|
|
],
|
|
"requires": [
|
|
"route"
|
|
],
|
|
"contributes": {
|
|
"route": {
|
|
"api": {
|
|
"label": "files-api",
|
|
"port": 9000
|
|
},
|
|
"console": {
|
|
"label": "files",
|
|
"port": 9001
|
|
}
|
|
}
|
|
},
|
|
"capabilities": [
|
|
"container-runtime"
|
|
],
|
|
"emits": [
|
|
"module.minio.bucket.created",
|
|
"module.minio.bucket.removed"
|
|
],
|
|
"listens": [
|
|
{
|
|
"port": 9000,
|
|
"protocol": "tcp",
|
|
"from": "mesh",
|
|
"why": "the S3 endpoint"
|
|
},
|
|
{
|
|
"port": 9001,
|
|
"protocol": "tcp",
|
|
"from": "mesh",
|
|
"why": "the admin console"
|
|
}
|
|
],
|
|
"serves": {
|
|
"s3-bucket": {
|
|
"scheme": "http",
|
|
"region": "us-east-1",
|
|
"port": 9000
|
|
}
|
|
},
|
|
"receives": {
|
|
"s3-bucket": "/var/lib/minio/grants/mesh.json"
|
|
},
|
|
"grants": {
|
|
"s3-bucket": "/var/lib/minio/grants"
|
|
},
|
|
"own-secrets": {
|
|
"root": "/var/lib/minio/root.secret",
|
|
"broker": "/var/lib/mesh/minio/broker"
|
|
},
|
|
"resources": [
|
|
{
|
|
"id": "mesh-state",
|
|
"type": "directory",
|
|
"path": "/var/lib/mesh/minio",
|
|
"mode": "0700"
|
|
},
|
|
{
|
|
"id": "state",
|
|
"type": "directory",
|
|
"path": "/var/lib/minio",
|
|
"mode": "0700"
|
|
},
|
|
{
|
|
"id": "grants",
|
|
"type": "directory",
|
|
"path": "/var/lib/minio/grants",
|
|
"mode": "0700"
|
|
},
|
|
{
|
|
"id": "root-env",
|
|
"type": "file",
|
|
"path": "/var/lib/minio/root.env",
|
|
"mode": "0600",
|
|
"content": "MINIO_ROOT_USER=meshroot\n"
|
|
},
|
|
{
|
|
"id": "data",
|
|
"type": "directory",
|
|
"path": "/var/lib/minio-store",
|
|
"mode": "0700"
|
|
},
|
|
{
|
|
"id": "net",
|
|
"type": "network",
|
|
"name": "minio-net"
|
|
},
|
|
{
|
|
"id": "server",
|
|
"type": "container",
|
|
"name": "minio",
|
|
"image": "docker.io/pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372",
|
|
"network": "minio-net",
|
|
"args": ["server", "/data", "--console-address", ":9001"],
|
|
"env-file": ["/var/lib/minio/root.env"],
|
|
"ports": ["9000", "9001"],
|
|
"volumes": [
|
|
"/var/lib/minio-store:/data",
|
|
"/var/lib/minio/root.secret:/run/secrets/root:ro"
|
|
],
|
|
"env": {
|
|
"MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root",
|
|
"MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be"
|
|
}
|
|
},
|
|
{
|
|
"id": "runtime",
|
|
"type": "container",
|
|
"name": "mesh-minio",
|
|
"network": "minio-net",
|
|
"volumes": [
|
|
"/var/lib/mesh/minio/broker:/run/secrets/broker:ro",
|
|
"/var/lib/minio/grants:/var/lib/minio/grants:ro",
|
|
"/var/lib/minio/root.secret:/run/secrets/root:ro"
|
|
],
|
|
"env": {
|
|
"MESH_MINIO_ENDPOINT": "http://minio:9000",
|
|
"MESH_MINIO_ROOT_USER": "meshroot",
|
|
"MESH_MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root",
|
|
"MESH_BROKER_FILE": "/run/secrets/broker",
|
|
"MESH_RECEIVES": "/var/lib/minio/grants/mesh.json"
|
|
},
|
|
"artifact": "runtime"
|
|
}
|
|
],
|
|
"build": {
|
|
"on": [
|
|
{
|
|
"arg": "BUILD_BASE",
|
|
"module": "mesh-tools",
|
|
"artifact": "build"
|
|
},
|
|
{
|
|
"arg": "RUNTIME_BASE",
|
|
"module": "mesh-tools",
|
|
"artifact": "runtime"
|
|
}
|
|
],
|
|
"artifacts": [
|
|
{
|
|
"name": "runtime",
|
|
"kind": "image",
|
|
"from": "Dockerfile"
|
|
}
|
|
]
|
|
}
|
|
}
|