From 4ac12115bacaab59392548ddf106871771baf100 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 3 Sep 2026 00:15:46 +0200 Subject: [PATCH] The catalogue moves out, to novox/mesh-catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The modules the mesh runs were under examples/modules/, which framed the real catalogue as illustrations of a control-plane package. They are neither examples nor the control plane's — they are the mesh's own catalogue, and they now live in their own repository (novox/mesh-catalog), consumed as a build source like any other. The engine that reads them stays here (internal/catalogue): the control plane owns the manifest contract; the data does not belong beside it. Removed with them: modules_test.go and parseall_test.go, which validated the example manifests against the parser. That validation logically follows the catalogue to mesh-catalog, but it imports internal/catalogue, so re-homing it needs the parser exported from internal/ first — a deliberate follow-up, not done here. Until then the pipeline is the gate, and internal/catalogue's own inline tests still cover the parser. Answers novox/hq ADR 0030's open tier-4 question — where the catalogue lives — in favour of one flat mesh-catalog repository. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- examples/modules/README.md | 75 --- examples/modules/bazarr.json | 73 --- examples/modules/distribution.json | 52 -- examples/modules/dnsmasq.json | 50 -- examples/modules/gitea.json | 81 --- examples/modules/grafana.json | 55 -- examples/modules/home-assistant.json | 37 -- examples/modules/icecast.json | 47 -- examples/modules/influxdb.json | 64 --- examples/modules/jackett.json | 41 -- examples/modules/keycloak.json | 81 --- examples/modules/mailu.json | 274 ---------- examples/modules/minio.json | 107 ---- examples/modules/modules_test.go | 652 ----------------------- examples/modules/nextcloud.json | 75 --- examples/modules/nodered.json | 39 -- examples/modules/nzbget.json | 49 -- examples/modules/ombi.json | 41 -- examples/modules/photos.json | 40 -- examples/modules/plex.json | 87 --- examples/modules/portainer.json | 36 -- examples/modules/postgres.json | 101 ---- examples/modules/qbittorrent.json | 49 -- examples/modules/radarr.json | 57 -- examples/modules/redis.json | 100 ---- examples/modules/resolv-conf.json | 12 - examples/modules/resolved-split-dns.json | 18 - examples/modules/searxng.json | 69 --- examples/modules/sonarr.json | 65 --- examples/modules/tautulli.json | 41 -- examples/modules/umami.json | 59 -- examples/modules/verdaccio.json | 51 -- internal/catalogue/parseall_test.go | 23 - 33 files changed, 2701 deletions(-) delete mode 100644 examples/modules/README.md delete mode 100644 examples/modules/bazarr.json delete mode 100644 examples/modules/distribution.json delete mode 100644 examples/modules/dnsmasq.json delete mode 100644 examples/modules/gitea.json delete mode 100644 examples/modules/grafana.json delete mode 100644 examples/modules/home-assistant.json delete mode 100644 examples/modules/icecast.json delete mode 100644 examples/modules/influxdb.json delete mode 100644 examples/modules/jackett.json delete mode 100644 examples/modules/keycloak.json delete mode 100644 examples/modules/mailu.json delete mode 100644 examples/modules/minio.json delete mode 100644 examples/modules/modules_test.go delete mode 100644 examples/modules/nextcloud.json delete mode 100644 examples/modules/nodered.json delete mode 100644 examples/modules/nzbget.json delete mode 100644 examples/modules/ombi.json delete mode 100644 examples/modules/photos.json delete mode 100644 examples/modules/plex.json delete mode 100644 examples/modules/portainer.json delete mode 100644 examples/modules/postgres.json delete mode 100644 examples/modules/qbittorrent.json delete mode 100644 examples/modules/radarr.json delete mode 100644 examples/modules/redis.json delete mode 100644 examples/modules/resolv-conf.json delete mode 100644 examples/modules/resolved-split-dns.json delete mode 100644 examples/modules/searxng.json delete mode 100644 examples/modules/sonarr.json delete mode 100644 examples/modules/tautulli.json delete mode 100644 examples/modules/umami.json delete mode 100644 examples/modules/verdaccio.json delete mode 100644 internal/catalogue/parseall_test.go diff --git a/examples/modules/README.md b/examples/modules/README.md deleted file mode 100644 index e62cf03..0000000 --- a/examples/modules/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Example modules - -Manifests, not programs. They are here because the contract is easier to read as something that -works than as a description of something that would. - -**Third-party software runs *on* the mesh, not *of* it** (novox/hq ADR 0001). dnsmasq is not the -mesh's, and neither is systemd-resolved — what is the mesh's is the fact only it can know, which -is which machines exist and where they are. So the mesh writes that to a file and these read it. - -## Resolving a service named under a machine - -`postgres.novox.internal`, `plex.ace.internal`. The first label is the service and the rest is the -node, so **anything under a node's name must resolve to that node** and a proxy there routes by -the name it was asked for. That routing is a separate concern and stays separate. - -Two roles, and they are genuinely different things: - -| | claims | | -|---|---|---| -| **serving** | `the-dns-port` | answers the wildcards — `dnsmasq.json` | -| **asking** | `the-resolver-configuration` | decides what the machine asks — `resolved-split-dns.json`, `resolv-conf.json` | - -**systemd-resolved cannot serve a wildcard**, so it is only ever an *asking* module: it routes the -mesh's suffix to something that can. Treating the two roles as one would produce a module that -cannot work, which is the mistake worth naming. - -Assign one of each. Two of either is refused by the mesh rather than fought over on the machine: - -> `resolved-split-dns and resolv-conf both claim "the-resolver-configuration", and only one thing -> may hold it per node` - -ADR 0009's table names that resource `/etc/resolv.conf`, which is what it *is*, the way it writes -*the seat*. A claim is a name in the catalogue's own form, so it is written as one. - -## Why neither needs to know the machine's address - -Both would ordinarily need it — a resolver must bind somewhere, and a stub must be pointed -somewhere — and a static manifest cannot know it. - -Neither does, because both name things **the mesh itself named**: the private network's interface -is `mesh0` on every machine, and the address a resolver listens on for the machine's own use is -`127.0.0.54` on every machine. A name the mesh chose is a name a manifest can use. - -## Asking for a bucket - -`minio.json` provides one, `photos.json` asks for one. Together they are the whole of an -edge, and they are here as a **pair** because that is the only way to see the halves line up: - -| the provider says | the consumer says | -|---|---| -| `provides: s3-bucket` | `requires: s3-bucket` | -| `receives` — where to be told who asked | `contributes: {bucket: photos}` — what it wants | -| `grants` — where their credentials land | `secrets` — where to be given its key | -| `serves` — port, scheme, region | `binds` — where to be told all that | - -**The mesh adds the half neither can know**: which machine the provider is on, and where it is on -the private network. Neither manifest names an address, and that is what lets the same pair work -on any mesh. - -**`s3-bucket` names the protocol, not the product** (novox/hq ADR 0027). A -consumer's code is written against the S3 API, and swapping one store for another does not break -it — so the coupling is to S3. A database is the other case: an application is written against -PostgreSQL or against SQL Server, so those provisions name the engine. - -**What the pair is checked for.** That the names match, that each side says where it wants to be -told, and that the consumer contributes the key the provisioner actually reads — `bucket`, not -`name`. Contributing `name` (which is what a database consumer contributes) resolves perfectly and -then fails on the machine with *asked for a bucket and did not name it*, which is a long way from -the manifest that caused it. - -The provisioner that makes the credential true lives in -[`../objectstore-provisioner`](../objectstore-provisioner), and is proven against a real store in -the lab — including the assertion a database does not need, that **a consumer cannot reach another -consumer's bucket**. One store holds every bucket behind one endpoint, so that isolation is a -policy somebody wrote rather than a boundary the product has. diff --git a/examples/modules/bazarr.json b/examples/modules/bazarr.json deleted file mode 100644 index 90e6ac2..0000000 --- a/examples/modules/bazarr.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "module": "bazarr", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 6767, - "protocol": "tcp", - "from": "mesh", - "why": "managing subtitles" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/bazarr/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "media-movies", - "type": "directory", - "path": "/services/media/movies", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-series", - "type": "directory", - "path": "/services/media/series", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-anime", - "type": "directory", - "path": "/services/media/anime", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-downloads", - "type": "directory", - "path": "/services/media/downloads", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "bazarr", - "image": "lscr.io/linuxserver/bazarr@sha256:3a820372f19fcb2981ea19fe4b5382934d67414afaba974bce831ddda0a64a02", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "6767" - ], - "volumes": [ - "/services/bazarr/config:/config", - "/services/media/movies:/movies", - "/services/media/series:/series", - "/services/media/anime:/anime", - "/services/media/downloads:/downloads" - ] - } - ] -} diff --git a/examples/modules/distribution.json b/examples/modules/distribution.json deleted file mode 100644 index 5cfb4ba..0000000 --- a/examples/modules/distribution.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "module": "distribution", - "version": "1", - "provides": [ - { - "name": "artifact-store", - "scope": "mesh" - } - ], - "claims": [ - { - "name": "the-artifact-store", - "scope": "node" - } - ], - "capabilities": [ - "container-runtime" - ], - "serves": { - "artifact-store": { - "port": 5000 - } - }, - "listens": [ - { - "port": 5000, - "protocol": "tcp", - "from": "mesh", - "why": "every machine pulls images and artifacts from here" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/mesh/registry", - "mode": "0700" - }, - { - "id": "store", - "type": "container", - "name": "mesh-registry", - "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", - "ports": [ - "5000:5000" - ], - "volumes": [ - "mesh-registry-data:/var/lib/registry" - ] - } - ] -} diff --git a/examples/modules/dnsmasq.json b/examples/modules/dnsmasq.json deleted file mode 100644 index 87ec177..0000000 --- a/examples/modules/dnsmasq.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "module": "dnsmasq", - "version": "1", - "provides": [ - "wildcard-resolution" - ], - "claims": [ - { - "name": "the-dns-port", - "scope": "node" - } - ], - "listens": [ - { - "port": 53, - "protocol": "udp", - "from": "mesh", - "why": "names under every machine in this mesh, for this machine and what it runs", - "fixed": true - } - ], - "resources": [ - { - "id": "package", - "type": "package", - "package": "dnsmasq" - }, - { - "id": "config", - "type": "file", - "path": "/etc/dnsmasq.conf", - "mode": "0644", - "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it \u2014 including a container\n# on this machine \u2014 can ask.\n# 127.0.0.55 this machine's own use, for whatever points resolution at the\n# mesh. Not .53 or .54: systemd-resolved holds BOTH \u2014 .53 is its\n# stub and .54 its proxy stub \u2014 which this module asserted was\n# free until a machine said otherwise.\n#\n# Listening on a loopback address makes dnsmasq take the rest of\n# loopback with it, 127.0.0.1 included. That is why this module\n# claims `the-dns-port`: it takes the machine's DNS port, and\n# saying it takes only one address would be the same kind of\n# comfortable claim that .54 was free.\n#\n# .55 is a convention and not a reservation. If a future systemd\n# takes it, this line changes and nothing else does, which is the\n# reason it is written once here rather than in each module that\n# points at it.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.55\n\n# **It forwards nothing, and must not read resolv.conf to find out where to.**\n# Whatever points this machine at the mesh writes its own address into\n# resolv.conf \u2014 so a resolver that read it for upstreams would find itself,\n# and every query it could not answer locally would loop until its receive\n# queue filled. That is not theoretical: it filled with 15KB of queries and\n# every lookup on the machine hung.\n#\n# It needs no upstream because it is never asked for anything else: the\n# asking module routes only the mesh's suffix here and leaves the rest\n# wherever the machine already sent it.\nno-resolv\ndomain-needed\nbogus-priv\n" - }, - { - "id": "service", - "type": "service", - "unit": "dnsmasq.service", - "state": "running", - "boot": "enabled", - "restart-on": [ - "config", - "dnsmasq.fact-node-zones" - ] - } - ], - "facts": { - "node-zones": "/etc/mesh-resolver/nodes.conf" - } -} diff --git a/examples/modules/gitea.json b/examples/modules/gitea.json deleted file mode 100644 index feb28b8..0000000 --- a/examples/modules/gitea.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "module": "gitea", - "version": "1", - "requires": [ - "postgres-database" - ], - "contributes": { - "postgres-database": { - "name": "gitea" - } - }, - "binds": { - "postgres-database": "/var/lib/gitea/database.json" - }, - "secrets": { - "postgres-database": "/var/lib/gitea/database.secret" - }, - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 3000, - "protocol": "tcp", - "from": "mesh", - "why": "the forge, over http" - }, - { - "port": 2222, - "protocol": "tcp", - "from": "mesh", - "why": "git over ssh. Not 22: the machine's own daemon holds that, and a module does not take it" - } - ], - "own-secrets": { - "internal-token": "/var/lib/gitea/internal-token.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/gitea", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/gitea/server.env", - "mode": "0600", - "content": "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\nGITEA__database__DB_TYPE=postgres\nGITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nGITEA__database__NAME=gitea\nGITEA__database__USER=${bound:postgres-database:as}\nGITEA__database__PASSWD=${secret:postgres-database}\n" - }, - { - "id": "data", - "type": "directory", - "path": "/services/gitea/gitea", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "gitea", - "image": "gitea/gitea@sha256:dfc61e347c8b582df918f4556401bf2cecdfbdb56c5282ae9488dd76fca3e41c", - "env": { - "DB_TYPE": "postgres", - "USER_UID": "1000", - "USER_GID": "1000" - }, - "env-file": [ - "/var/lib/gitea/server.env" - ], - "ports": [ - "3000", - "2222:22" - ], - "volumes": [ - "/services/gitea/gitea:/data" - ] - } - ] -} diff --git a/examples/modules/grafana.json b/examples/modules/grafana.json deleted file mode 100644 index 5dcbbd5..0000000 --- a/examples/modules/grafana.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "module": "grafana", - "version": "1", - "own-secrets": { - "admin": "/var/lib/grafana-module/admin.secret" - }, - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 3000, - "protocol": "tcp", - "from": "mesh", - "why": "the dashboards. Also 3000 inside, like the forge - which is the mesh's port assignment earning its keep" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/grafana-module", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/grafana-module/server.env", - "mode": "0600", - "content": "GF_SECURITY_ADMIN_PASSWORD=${secret:admin}\n" - }, - { - "id": "data", - "type": "directory", - "path": "/services/grafana/data", - "mode": "0700", - "owner": "472:472" - }, - { - "id": "server", - "type": "container", - "name": "grafana", - "image": "grafana/grafana@sha256:f772d434e8fab0049deb2b1b30abd43342bcfca1537614aa8d36080232cf4283", - "env-file": [ - "/var/lib/grafana-module/server.env" - ], - "ports": [ - "3000" - ], - "volumes": [ - "/services/grafana/data:/var/lib/grafana" - ] - } - ] -} diff --git a/examples/modules/home-assistant.json b/examples/modules/home-assistant.json deleted file mode 100644 index 6003229..0000000 --- a/examples/modules/home-assistant.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "module": "home-assistant", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 8123, - "protocol": "tcp", - "from": "mesh", - "why": "the dashboard and the API" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/home-assistant/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "home-assistant", - "image": "ghcr.io/home-assistant/home-assistant@sha256:14931c6b13756317849f46da1d01b45937a1150db66c081cfe529d48215943fe", - "network": "host", - "env": { - "TZ": "Etc/UTC" - }, - "volumes": [ - "/services/home-assistant/config:/config" - ] - } - ] -} diff --git a/examples/modules/icecast.json b/examples/modules/icecast.json deleted file mode 100644 index 77d208e..0000000 --- a/examples/modules/icecast.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "module": "icecast", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "own-secrets": { - "source": "/var/lib/icecast-module/source.secret", - "admin": "/var/lib/icecast-module/admin.secret", - "relay": "/var/lib/icecast-module/relay.secret" - }, - "listens": [ - { - "port": 8000, - "protocol": "tcp", - "from": "mesh", - "why": "streams in from sources and out to listeners" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/icecast-module", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/icecast-module/server.env", - "mode": "0600", - "content": "ICECAST_SOURCE_PASSWORD=${secret:source}\nICECAST_ADMIN_PASSWORD=${secret:admin}\nICECAST_RELAY_PASSWORD=${secret:relay}\nICECAST_ADMIN_USERNAME=admin\n" - }, - { - "id": "server", - "type": "container", - "name": "icecast", - "image": "infiniteproject/icecast@sha256:cd506cf3dfe31ce05fd37d7e672dbd1213e7255cc93d28ecf5a3b547af4e162c", - "env-file": [ - "/var/lib/icecast-module/server.env" - ], - "ports": [ - "8000" - ] - } - ] -} diff --git a/examples/modules/influxdb.json b/examples/modules/influxdb.json deleted file mode 100644 index ad64e0c..0000000 --- a/examples/modules/influxdb.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "module": "influxdb", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "own-secrets": { - "admin": "/var/lib/influxdb-module/admin.secret", - "admin-token": "/var/lib/influxdb-module/admin-token.secret" - }, - "listens": [ - { - "port": 8086, - "protocol": "tcp", - "from": "mesh", - "why": "queries and writes, over http" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/influxdb-module", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/influxdb-module/server.env", - "mode": "0600", - "content": "DOCKER_INFLUXDB_INIT_MODE=setup\nDOCKER_INFLUXDB_INIT_USERNAME=admin\nDOCKER_INFLUXDB_INIT_PASSWORD=${secret:admin}\nDOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${secret:admin-token}\nDOCKER_INFLUXDB_INIT_ORG=mesh\nDOCKER_INFLUXDB_INIT_BUCKET=default\n" - }, - { - "id": "data", - "type": "directory", - "path": "/services/influxdb/data", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "config", - "type": "directory", - "path": "/services/influxdb/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "influxdb", - "image": "influxdb@sha256:f75e48af0598e8aec7986e991a848d19a119101a7d563a2e5db1dfaac9c45daa", - "env-file": [ - "/var/lib/influxdb-module/server.env" - ], - "ports": [ - "8086" - ], - "volumes": [ - "/services/influxdb/data:/var/lib/influxdb2", - "/services/influxdb/config:/etc/influxdb2" - ] - } - ] -} diff --git a/examples/modules/jackett.json b/examples/modules/jackett.json deleted file mode 100644 index f1e4a74..0000000 --- a/examples/modules/jackett.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "module": "jackett", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 9117, - "protocol": "tcp", - "from": "mesh", - "why": "the indexer proxy" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/jackett/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "jackett", - "image": "lscr.io/linuxserver/jackett@sha256:fd72d42b731ebf750b5de9711127251cf3b3f609419c32083ea8b3b3ee840b77", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "9117" - ], - "volumes": [ - "/services/jackett/config:/config" - ] - } - ] -} diff --git a/examples/modules/keycloak.json b/examples/modules/keycloak.json deleted file mode 100644 index 5096b20..0000000 --- a/examples/modules/keycloak.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "module": "keycloak", - "version": "1", - "requires": [ - "postgres-database" - ], - "contributes": { - "postgres-database": { - "name": "keycloak" - } - }, - "binds": { - "postgres-database": "/var/lib/keycloak/database.json" - }, - "secrets": { - "postgres-database": "/var/lib/keycloak/database.secret" - }, - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 8080, - "protocol": "tcp", - "from": "mesh", - "why": "anything the mesh runs that authenticates a person" - } - ], - "own-secrets": { - "admin": "/var/lib/keycloak/admin.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/keycloak", - "mode": "0700" - }, - { - "id": "admin-env", - "type": "file", - "path": "/var/lib/keycloak/admin.env", - "mode": "0600", - "content": "KEYCLOAK_ADMIN=admin\nKEYCLOAK_ADMIN_PASSWORD=${secret:admin}\n" - }, - { - "id": "database-env", - "type": "file", - "path": "/var/lib/keycloak/database.env", - "mode": "0600", - "content": "KC_DB_URL=jdbc:postgresql://${bound:postgres-database:at}:${bound:postgres-database:port}/keycloak\nKC_DB_USERNAME=${bound:postgres-database:as}\nKC_DB_PASSWORD=${secret:postgres-database}\n" - }, - { - "id": "net", - "type": "network", - "name": "keycloak" - }, - { - "id": "server", - "type": "container", - "name": "keycloak", - "image": "quay.io/keycloak/keycloak@sha256:ecd43971114b0c764f8a3288dddab73f98cb473daccc4feaffe4dc14adeaf866", - "network": "keycloak", - "args": [ - "start-dev" - ], - "env": { - "KC_DB": "postgres", - "KC_HTTP_ENABLED": "true", - "KC_HEALTH_ENABLED": "true" - }, - "env-file": [ - "/var/lib/keycloak/admin.env", - "/var/lib/keycloak/database.env" - ], - "ports": [ - "8080" - ] - } - ] -} diff --git a/examples/modules/mailu.json b/examples/modules/mailu.json deleted file mode 100644 index b1fe323..0000000 --- a/examples/modules/mailu.json +++ /dev/null @@ -1,274 +0,0 @@ -{ - "module": "mailu", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 25, - "protocol": "tcp", - "from": "anywhere", - "why": "mail from other mail servers", - "fixed": true - }, - { - "port": 465, - "protocol": "tcp", - "from": "anywhere", - "why": "submission over TLS", - "fixed": true - }, - { - "port": 587, - "protocol": "tcp", - "from": "anywhere", - "why": "submission", - "fixed": true - }, - { - "port": 993, - "protocol": "tcp", - "from": "anywhere", - "why": "IMAP over TLS", - "fixed": true - }, - { - "port": 7080, - "protocol": "tcp", - "from": "mesh", - "why": "the web interface, behind a proxy" - } - ], - "own-secrets": { - "secret-key": "/var/lib/mailu/secret-key.secret", - "database": "/var/lib/mailu/database.secret", - "admin": "/var/lib/mailu/admin.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/mailu", - "mode": "0700" - }, - { - "id": "secret-env", - "type": "file", - "path": "/var/lib/mailu/secret.env", - "mode": "0600", - "content": "SECRET_KEY=${secret:secret-key}\n" - }, - { - "id": "database-env", - "type": "file", - "path": "/var/lib/mailu/database.env", - "mode": "0600", - "content": "POSTGRES_USER=mailu\nPOSTGRES_DB=mailu\nPOSTGRES_PASSWORD=${secret:database}\nDB_USER=mailu\nDB_NAME=mailu\nDB_PW=${secret:database}\n" - }, - { - "id": "admin-env", - "type": "file", - "path": "/var/lib/mailu/admin.env", - "mode": "0600", - "content": "INITIAL_ADMIN_PW=${secret:admin}\n" - }, - { - "id": "data-certs", - "type": "directory", - "path": "/services/mailu/data/certs", - "mode": "0700" - }, - { - "id": "data-data", - "type": "directory", - "path": "/services/mailu/data/data", - "mode": "0700" - }, - { - "id": "data-dkim", - "type": "directory", - "path": "/services/mailu/data/dkim", - "mode": "0700" - }, - { - "id": "data-filter", - "type": "directory", - "path": "/services/mailu/data/filter", - "mode": "0700" - }, - { - "id": "data-mail", - "type": "directory", - "path": "/services/mailu/data/mail", - "mode": "0700" - }, - { - "id": "data-mailqueue", - "type": "directory", - "path": "/services/mailu/data/mailqueue", - "mode": "0700" - }, - { - "id": "data-redis", - "type": "directory", - "path": "/services/mailu/data/redis", - "mode": "0700" - }, - { - "id": "data-webmail", - "type": "directory", - "path": "/services/mailu/data/webmail", - "mode": "0700" - }, - { - "id": "data-dovecot", - "type": "directory", - "path": "/services/mailu/data/overrides/dovecot", - "mode": "0700" - }, - { - "id": "data-nginx", - "type": "directory", - "path": "/services/mailu/data/overrides/nginx", - "mode": "0700" - }, - { - "id": "data-pgdata", - "type": "directory", - "path": "/services/mailu/data/data/psql_admindb/pgdata", - "mode": "0700" - }, - { - "id": "net", - "type": "network", - "name": "mailu" - }, - { - "id": "resolver", - "type": "container", - "name": "mailu-resolver", - "image": "ghcr.io/mailu/unbound@sha256:142aaad82ad1b0d5b59a5f1303778dba61a3e0a540f5d969c48862bcc99f6f5d", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env" - ] - }, - { - "id": "redis", - "type": "container", - "name": "mailu-redis", - "image": "redis@sha256:1db42ccef14898aa29bae778452d567534b59c107129cbc1163fb552de184d3c", - "network": "mailu", - "volumes": [ - "/services/mailu/data/redis:/data" - ] - }, - { - "id": "admindb", - "type": "container", - "name": "mailu-admindb", - "image": "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee", - "network": "mailu", - "env": { - "PGDATA": "/var/lib/postgresql/data/pgdata" - }, - "env-file": [ - "/var/lib/mailu/database.env" - ], - "volumes": [ - "/services/mailu/data/data/psql_admindb/pgdata:/var/lib/postgresql/data/pgdata" - ] - }, - { - "id": "admin", - "type": "container", - "name": "mailu-admin", - "image": "ghcr.io/mailu/admin@sha256:dcac20e9cbdad560faef9653b1b5ac0d9266f4098dc00f0e7f0d35f4e70ed8f1", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env", - "/var/lib/mailu/database.env", - "/var/lib/mailu/admin.env" - ], - "volumes": [ - "/services/mailu/data/data:/data", - "/services/mailu/data/dkim:/dkim" - ] - }, - { - "id": "imap", - "type": "container", - "name": "mailu-imap", - "image": "ghcr.io/mailu/dovecot@sha256:46d18ba51032be8ebd6841aa49c1ef8762c729038c5fd86a081b5b884d478af9", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env" - ], - "volumes": [ - "/services/mailu/data/mail:/mail", - "/services/mailu/data/overrides/dovecot:/overrides:ro" - ] - }, - { - "id": "smtp", - "type": "container", - "name": "mailu-smtp", - "image": "ghcr.io/mailu/postfix@sha256:bbf882880f68849511710b35237a933f3fe80c4b28bf48ff20205dbd1f1433d7", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env" - ], - "volumes": [ - "/services/mailu/data/mailqueue:/queue" - ] - }, - { - "id": "antispam", - "type": "container", - "name": "mailu-antispam", - "image": "ghcr.io/mailu/rspamd@sha256:e87ab93dd252cc69499caa5317dd10d445fd4291a7ecf6bca09793c7d475a0c8", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env" - ], - "volumes": [ - "/services/mailu/data/filter:/var/lib/rspamd" - ] - }, - { - "id": "webmail", - "type": "container", - "name": "mailu-webmail", - "image": "ghcr.io/mailu/webmail@sha256:076b720fc766e58a97321cdb700e887c2008d6d323685fe59f323088333059dc", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env" - ], - "volumes": [ - "/services/mailu/data/webmail:/data" - ] - }, - { - "id": "front", - "type": "container", - "name": "mailu-front", - "image": "ghcr.io/mailu/nginx@sha256:09f28ab6d36367fcacc7994f7021f132ac845bdc05f04bf80906102d11aaa057", - "network": "mailu", - "env-file": [ - "/var/lib/mailu/secret.env" - ], - "ports": [ - "25", - "465", - "587", - "993", - "7080:80" - ], - "volumes": [ - "/services/mailu/data/certs:/certs", - "/services/mailu/data/overrides/nginx:/overrides:ro" - ] - } - ] -} diff --git a/examples/modules/minio.json b/examples/modules/minio.json deleted file mode 100644 index 12c285a..0000000 --- a/examples/modules/minio.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "module": "minio", - "version": "1", - "provides": [ - { - "name": "s3-bucket", - "scope": "mesh" - } - ], - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 9000, - "protocol": "tcp", - "from": "mesh", - "why": "the S3 endpoint" - } - ], - "serves": { - "s3-bucket": { - "scheme": "http", - "region": "us-east-1" - } - }, - "receives": { - "s3-bucket": "/var/lib/minio/grants/mesh.json" - }, - "grants": { - "s3-bucket": "/var/lib/minio/grants" - }, - "own-secrets": { - "root": "/var/lib/minio/root.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/minio", - "mode": "0700" - }, - { - "id": "grants", - "type": "directory", - "path": "/var/lib/minio/grants", - "mode": "0700" - }, - { - "id": "root-env", - "type": "file", - "path": "/var/lib/minio/root.env", - "mode": "0600", - "content": "MINIO_ROOT_USER=meshroot\nMINIO_ROOT_PASSWORD=${secret:root}\n" - }, - { - "id": "data", - "type": "directory", - "path": "/services/minio/data/data1-1", - "mode": "0700" - }, - { - "id": "net", - "type": "network", - "name": "minio" - }, - { - "id": "server", - "type": "container", - "name": "minio", - "image": "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2", - "network": "minio", - "args": [ - "server", - "/data", - "--console-address", - ":9001" - ], - "env-file": [ - "/var/lib/minio/root.env" - ], - "ports": [ - "9000" - ], - "volumes": [ - "/services/minio/data/data1-1:/data" - ] - }, - { - "id": "provisioner", - "type": "container", - "name": "mesh-provision-objectstore", - "image": "mesh-provision-objectstore@sha256:0000000000000000000000000000000000000000000000000000000000000000", - "network": "minio", - "env": { - "GRANTS": "/var/lib/minio/grants", - "MESH_OBJECTSTORE_URL": "http://minio:9000", - "MESH_OBJECTSTORE_ROOT_USER": "meshroot", - "MESH_OBJECTSTORE_ROOT_PASSWORD_FILE": "/run/secrets/root" - }, - "volumes": [ - "/var/lib/minio/grants:/var/lib/minio/grants:ro", - "/var/lib/minio/root.secret:/run/secrets/root:ro" - ] - } - ] -} diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go deleted file mode 100644 index bd60453..0000000 --- a/examples/modules/modules_test.go +++ /dev/null @@ -1,652 +0,0 @@ -package modules - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "slices" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/overlay" -) - -// The examples are manifests, so the thing to check is that the catalogue accepts them. -// -// A manifest that only ever appears in a document is a manifest nobody has run through the parser, -// and the parser refuses unknown keys — so a typo here would be discovered by whoever first tried -// to use one, which is the opposite of what an example is for. -func read(t *testing.T, name string) catalogue.Manifest { - t.Helper() - raw, err := os.ReadFile(filepath.Join(".", name)) - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - t.Fatalf("%s is not a manifest this mesh accepts: %v", name, err) - } - return m -} - -func TestEveryExampleIsAManifestTheMeshAccepts(t *testing.T) { - found, err := filepath.Glob("*.json") - if err != nil { - t.Fatal(err) - } - if len(found) == 0 { - t.Fatal("no examples, so this test proves nothing") - } - for _, name := range found { - read(t, name) - } -} - -// The serving module reads what the mesh writes, and restarts when the mesh rewrites it. -// -// Without the second it would serve the names it started with for ever — every machine that -// joined afterwards unreachable by name, and every check passing. -func TestTheResolverReadsTheMeshsNamesAndFollowsThem(t *testing.T) { - m := read(t, "dnsmasq.json") - - var config, service map[string]any - for _, r := range m.Resources { - switch r["id"] { - case "config": - config = r - case "service": - service = r - } - } - if config == nil || service == nil { - t.Fatal("the module has no configuration or no service") - } - // The zone file is a FACT the module asks for, at a path it chose. The mesh writes it there; - // what reads it and how is this module's own business, which is the whole shape. - const zones = "/etc/mesh-resolver/nodes.conf" - if !strings.Contains(config["content"].(string), zones) { - t.Fatalf("it does not read what the mesh writes at %s", zones) - } - - var follows bool - for _, id := range service["restart-on"].([]any) { - if id.(string) == "dnsmasq.fact-node-zones" { - follows = true - } - } - if !follows { - t.Fatalf("it does not restart when the mesh rewrites the names: %v", service["restart-on"]) - } -} - -// It binds names the mesh chose, so it needs to know nothing about the machine it is on. -// -// That is the whole reason these can be static manifests: a resolver must bind somewhere and a -// stub must be pointed somewhere, and neither address is knowable in advance — unless the mesh -// named it. -func TestTheResolverNeedsToKnowNothingAboutItsMachine(t *testing.T) { - config := read(t, "dnsmasq.json").Resources[1]["content"].(string) - // The directive, not the word: the comment above it names the interface too, so a plain - // Contains passes whatever the module actually binds. It did. - if !strings.Contains(config, "interface="+overlay.Interface+"\n") { - t.Fatalf("it does not bind the private network's interface %q:\n%s", - overlay.Interface, config) - } - // That it binds one, not which. Which address it is belongs in the manifests, where the - // asking modules can be checked against it — naming it here too would be a fourth place to - // keep in step, and the one nobody would think to change. - if !strings.Contains(config, "\nlisten-address=127.0.0.") { - t.Fatalf("it answers on no address for the machine's own use:\n%s", config) - } - // Not an address that belongs to something else. - // - // **systemd-resolved holds .53 AND .54** — the stub and the proxy stub. This module asserted - // .54 was free, in a comment that read as reasoned, and a machine said otherwise: dnsmasq - // could not start at all. A unit test cannot know which addresses a machine has spare, but it - // can hold on to what one has already told us. - for _, taken := range []string{"127.0.0.1", "127.0.0.53", "127.0.0.54"} { - if strings.Contains(config, "listen-address="+taken) { - t.Fatalf("it takes %s, which belongs to something else:\n%s", taken, config) - } - } -} - -// Everything that points resolution at the mesh points at the same place. -// -// Three files name this address — one binds it and two send queries to it — and a change to one -// of them alone is a resolver answering where nobody asks. -func TestTheAskingModulesPointAtWhereTheResolverAnswers(t *testing.T) { - serving := read(t, "dnsmasq.json").Resources[1]["content"].(string) - var at string - for _, line := range strings.Split(serving, "\n") { - if rest, found := strings.CutPrefix(strings.TrimSpace(line), "listen-address="); found { - at = rest - } - } - if at == "" { - t.Fatal("the resolver binds no address for the machine's own use") - } - for _, asking := range []string{"resolved-split-dns.json", "resolv-conf.json"} { - m := read(t, asking) - var mentions bool - for _, r := range m.Resources { - if content, ok := r["content"].(string); ok && strings.Contains(content, at) { - mentions = true - } - } - if !mentions { - t.Fatalf("%s does not point at %s, where the resolver answers", asking, at) - } - } -} - -// The two ways of deciding what a machine asks claim the same thing, so the mesh refuses the pair. -func TestTwoWaysOfOwningTheResolverCannotBothBeAssigned(t *testing.T) { - shelf := map[string]catalogue.Manifest{} - for _, name := range []string{"resolved-split-dns.json", "resolv-conf.json", "dnsmasq.json"} { - m := read(t, name) - shelf[m.Module] = m - } - // Something has to answer `wildcard-resolution`, or they are refused for that instead and the - // test would pass without ever reaching the claim. - shelf["dnsmasq"] = read(t, "dnsmasq.json") - - _, err := catalogue.Resolve(shelf, - []string{"dnsmasq", "resolved-split-dns", "resolv-conf"}, - catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, - catalogue.World{Unchecked: true}) - if err == nil { - t.Fatal("both ways of owning the resolver were assigned to one machine") - } - said := err.Error() - if !strings.Contains(said, "the-resolver-configuration") { - t.Fatalf("the refusal does not name what they both want: %v", said) - } -} - -// And the two roles are not the same claim: a machine runs one resolver AND one thing deciding -// what it asks, so serving and asking must be assignable together. -func TestServingAndAskingAreAssignableTogether(t *testing.T) { - shelf := map[string]catalogue.Manifest{} - for _, name := range []string{"dnsmasq.json", "resolved-split-dns.json"} { - m := read(t, name) - shelf[m.Module] = m - } - if _, err := catalogue.Resolve(shelf, - []string{"dnsmasq", "resolved-split-dns"}, - catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, - catalogue.World{Unchecked: true}); err != nil { - t.Fatalf("a resolver and the thing pointing at it cannot both be assigned: %v", err) - } -} - -// The examples are JSON a person edits, so a stray comma is worth catching here rather than on a -// machine. -func TestTheExamplesAreWellFormed(t *testing.T) { - found, _ := filepath.Glob("*.json") - for _, name := range found { - raw, err := os.ReadFile(name) - if err != nil { - t.Fatal(err) - } - var any map[string]any - if err := json.Unmarshal(raw, &any); err != nil { - t.Fatalf("%s is not JSON: %v", name, err) - } - } -} - -// The resolver must not look up its own upstreams. -// -// Whatever points a machine at the mesh writes that address into resolv.conf, so a resolver that -// read it would find itself — and every query it could not answer locally would loop until its -// receive queue filled. It did: 15KB of queries backed up and every lookup on the machine hung. -// -// It needs no upstream because it is never asked for anything else: the asking module routes only -// the mesh's suffix here and leaves the rest where the machine already sent it. -func TestTheResolverDoesNotAskItselfForUpstreams(t *testing.T) { - config := read(t, "dnsmasq.json").Resources[1]["content"].(string) - if !strings.Contains(config, "\nno-resolv\n") { - t.Fatalf("it reads resolv.conf for upstreams, which now points at itself:\n%s", config) - } - // And names no upstream of its own: choosing one would send every query this machine cannot - // answer somewhere nobody agreed to. - for _, line := range strings.Split(config, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "server=") { - t.Fatalf("it forwards to %q, which is not the mesh's to choose", line) - } - } -} - -// The two halves of an object-store edge, as a pair. -// -// `minio.json` is the provider. There were two manifests describing the same object store — the -// other named `object-store.json` — which is not a choice between implementations but one module -// written twice: same image, same provision, same scope. Assigning both to a node would have -// collided on `s3-bucket`. -// -// A provider and a consumer that only ever appear separately are two manifests nobody has checked -// against each other: the name one provides has to be the name the other requires, and the key a -// consumer contributes has to be the one the provisioner reads. Both were got wrong while writing -// them, and neither would have been caught by parsing either file alone. -func TestTheObjectStoreEdgeFitsTogether(t *testing.T) { - provider := read(t, "minio.json") - consumer := read(t, "photos.json") - - const provision = "s3-bucket" - - var provides bool - for _, offer := range provider.Provides { - if offer.Name == provision { - provides = true - } - } - if !provides { - t.Fatalf("the provider does not offer %q", provision) - } - if !strings.Contains(strings.Join(consumer.Requires, ","), provision) { - t.Fatalf("the consumer does not require %q", provision) - } - - // Where each side wants to be told. A provider that receives nowhere is a provider the mesh - // writes nothing for, and a provisioner with nothing to read. - if provider.Receives[provision] == "" { - t.Error("the provider says nowhere to write what its consumers asked for") - } - if provider.Grants[provision] == "" { - t.Error("the provider says nowhere to write its consumers' credentials") - } - if consumer.Binds[provision] == "" { - t.Error("the consumer says nowhere to be told where its bucket is") - } - if consumer.Secrets[provision] == "" { - t.Error("the consumer says nowhere to be given its key") - } - - // The key the provisioner reads out of `values`. It looks for `bucket`, so a consumer - // contributing `name` — which is what the database one contributes — resolves cleanly and - // then fails on the machine with "asked for a bucket and did not name it". - if _, named := consumer.Contributes[provision]["bucket"]; !named { - t.Errorf("the consumer contributes %v, and the provisioner reads \"bucket\"", - consumer.Contributes[provision]) - } -} - -// Every hole an example leaves for a credential can be filled from what that module declared. -// -// **A manifest that parses is not a manifest that works.** These say `${secret:x}` in a file and -// declare `x` under `own-secrets`; if the two ever disagree the mesh refuses the whole declaration -// at push time, on the machine, with the module's name and nothing else to go on. Checking it here -// costs nothing and moves the answer to whoever edited the file. -// -// This is also the shape that was missing entirely until 2026-09-01: an own secret arrives as a -// file whose whole content is the password, and every one of these programs reads `KEY=value`. The -// manifests said `own-secrets` pointed at a `.env` and it did not — it pointed at a password. -func TestEveryCredentialHoleCanBeFilledByTheModuleThatLeftIt(t *testing.T) { - found, err := filepath.Glob("*.json") - if err != nil { - t.Fatal(err) - } - var checked int - for _, name := range found { - m := read(t, name) - has := map[string]bool{} - for own := range m.OwnSecrets { - has[own] = true - } - for required := range m.Secrets { - has[required] = true - } - for _, r := range m.Resources { - content, ok := r["content"].(string) - if !ok { - continue - } - for _, wanted := range secretsUsedForTest(content) { - checked++ - if !has[wanted] { - t.Errorf( - "%s: %v says ${secret:%s}, and %s neither owns a secret by that name "+ - "nor requires anything that grants one", - name, r["id"], wanted, m.Module) - } - } - } - } - if checked == 0 { - t.Fatal("no example puts a credential into a file, so this test proves nothing") - } -} - -// A secret file is a password and nothing else, so nothing may read one as an env file. -// -// The fault this catches is the one these manifests shipped with: `own-secrets` pointing at a -// path called `.env`, mounted as `env-file`, holding a bare password. Docker reads that as a -// malformed line and the container starts with no password at all. -func TestNoContainerReadsABarePasswordAsAnEnvFile(t *testing.T) { - found, _ := filepath.Glob("*.json") - for _, name := range found { - m := read(t, name) - bare := map[string]bool{} - for _, where := range m.OwnSecrets { - bare[where] = true - } - for _, where := range m.Secrets { - bare[where] = true - } - for _, r := range m.Resources { - files, ok := r["env-file"].([]any) - if !ok { - continue - } - for _, f := range files { - if bare[fmt.Sprint(f)] { - t.Errorf( - "%s: %v reads %s as an env file, and that path holds a bare password — "+ - "declare a file whose content says ${secret:...} and read that instead", - name, r["id"], f) - } - } - } - } -} - -// The same expression the control plane and the host both match. -var placeholder = regexp.MustCompile(`\$\{secret:([a-z0-9][a-z0-9-]*)\}`) - -func secretsUsedForTest(content string) []string { - var used []string - seen := map[string]bool{} - for _, m := range placeholder.FindAllStringSubmatch(content, -1) { - if !seen[m[1]] { - seen[m[1]] = true - used = append(used, m[1]) - } - } - return used -} - -// Every module that requires something produces configuration a program could use. -// -// **Parsing is not working, and this file has now learned that twice.** These modules parsed and -// resolved for a day while their credentials went into files nothing could read; they would parse -// and resolve just as happily with a connection string naming no user, or with a placeholder -// written through as a hostname. What has to be true is that the bytes reaching the machine are -// usable, so that is what this asks — of every consumer, not of the one that was being worked on. -func TestEveryConsumerGetsConfigurationAProgramCouldUse(t *testing.T) { - found, err := filepath.Glob("*.json") - if err != nil { - t.Fatal(err) - } - shelf := map[string]catalogue.Manifest{} - for _, name := range found { - m := read(t, name) - shelf[m.Module] = m - } - - var checked int - for _, m := range shelf { - if len(m.Requires) == 0 { - continue - } - out := declareOnItsOwn(t, shelf, m) - if out == nil { - continue - } - checked++ - for _, r := range out { - content, ok := r["content"].(string) - if !ok { - continue - } - // A placeholder written through is read as a value by whatever parses the file — a - // connection to a host literally called "${bound:postgres-database:at}", failing - // somewhere that names neither the module nor the mesh. - if strings.Contains(content, "${bound:") { - t.Errorf("%s: %v reached the machine with a placeholder in it:\n%s", - m.Module, r["id"], content) - } - // The password is the one that must survive: only the host may fill it, and only on - // the machine. If it is gone, something composed it here. - for _, line := range strings.Split(content, "\n") { - if strings.Contains(line, "PASSWORD") || strings.Contains(line, "PASSWD") { - if !strings.Contains(line, "${secret:") { - t.Errorf("%s: %v carries %q, which is not a hole the host fills", - m.Module, r["id"], line) - } - } - } - } - } - if checked == 0 { - t.Fatal("no example requires anything, so this test proves nothing") - } -} - -// declareOnItsOwn resolves one consumer against a mesh that answers everything it requires, and -// returns what would reach the machine. Nil when its requirements cannot be answered from the -// examples, which is not this test's business to complain about. -func declareOnItsOwn(t *testing.T, shelf map[string]catalogue.Manifest, - m catalogue.Manifest) []map[string]any { - t.Helper() - - // Everything it requires, answered from somewhere else in the mesh, with whatever the - // providing example says it serves. - offered := map[string][]catalogue.Provider{} - for _, want := range m.Requires { - // Built the way the control plane builds it: what a provider tells a consumer includes - // the port, and the module no longer writes that into `serves` by hand — it says it once - // in `listens` and the mesh puts it there (novox/hq ADR 0038). - serves := map[string]any{} - for _, other := range shelf { - if _, said := other.Serves[want]; said { - serves = catalogue.ServedOn(other, want, nil) - } - } - offered[want] = []catalogue.Provider{ - {Node: "anchor", At: "anchor.internal", Serves: serves}} - } - resolved, err := catalogue.Resolve(shelf, []string{m.Module}, - catalogue.Node{Name: "workstation", At: "workstation.internal", - Capabilities: map[string]bool{"container-runtime": true}}, - catalogue.World{Offered: offered}) - if err != nil { - t.Logf("%s does not resolve on its own: %v", m.Module, err) - return nil - } - for i := range resolved.Needs { - resolved.Needs[i].Sealed = "sealed" - } - own := map[string]map[string]string{} - for name := range m.OwnSecrets { - if own[m.Module] == nil { - own[m.Module] = map[string]string{} - } - own[m.Module][name] = "sealed" - } - out, err := resolved.Declaration(catalogue.Rendering{Needed: own}) - if err != nil { - t.Errorf("%s resolves and does not declare: %v", m.Module, err) - return nil - } - return out -} - -// Every image an example names is one this repository builds. -// -// A manifest naming an image nothing produces is a module that resolves, plans, pushes, and stops -// on the machine at `docker pull` — the fault arriving as far from its cause as it can get. Two of -// these were found by reading the manifests rather than by running them: the object store's -// provisioner had a Dockerfile and no target, and Keycloak's did not exist at all. -// -// Only the mesh's own images are checked. `postgres`, `redis` and the rest come from a registry -// and are somebody else's to build; what this bounds is the set this repository is responsible -// for and might forget. -func TestEveryImageTheExamplesNameIsOneThisRepositoryBuilds(t *testing.T) { - makefile, err := os.ReadFile(filepath.Join("..", "..", "Makefile")) - if err != nil { - t.Fatal(err) - } - - found, _ := filepath.Glob("*.json") - var checked int - for _, name := range found { - for _, r := range read(t, name).Resources { - image, ok := r["image"].(string) - if !ok { - continue - } - repository, _, _ := strings.Cut(image, "@") - if !strings.HasPrefix(repository, "mesh-") { - continue - } - checked++ - if !strings.Contains(string(makefile), repository+":") { - t.Errorf( - "%s names the image %q and nothing in this repository builds one. A module "+ - "naming an image that does not exist resolves, plans, pushes, and stops "+ - "on the machine at `docker pull`", - name, repository) - } - } - } - if checked == 0 { - t.Fatal("no example names an image this repository builds, so this proves nothing") - } -} - -// Every host path a container mounts is a directory the module declared. -// -// **The mesh owns a directory or it does not** (novox/hq 04-ISSUES/026). A bind mount whose source -// does not exist is created by the container runtime as root, with a mode nobody chose — so -// `owner` and `mode` go unapplied on exactly the directories that hold the data. -// -// Worse, the rule that a directory is *kept* rather than removed when it holds something the mesh -// did not put there (ADR 0030) is written in terms of declared directories. An undeclared one is -// not covered by it. So the single rule guarding against data loss reached the configuration and -// not the data. -// -// These manifests were written by carrying compose files across, and a container shape that can -// express a compose file gets filled in like one. This is the check that says so. -func TestEveryMountedPathIsADirectoryTheModuleDeclared(t *testing.T) { - found, _ := filepath.Glob("*.json") - var checked int - for _, name := range found { - m := read(t, name) - declared := map[string]bool{} - for _, r := range m.Resources { - if fmt.Sprint(r["type"]) == "directory" { - declared[fmt.Sprint(r["path"])] = true - } - } - for _, r := range m.Resources { - for _, v := range stringsOfTest(r["volumes"]) { - host, _, _ := strings.Cut(v, ":") - if !strings.HasPrefix(host, "/") { - continue // a named volume, which the runtime owns and the mesh does not - } - checked++ - // A machine facility is not the module's data, and the manifest cannot yet say - // so (novox/hq 04-ISSUES/026, reopened on exactly this): the runtime's socket - // exists, the machine owns it, and declaring it as the module's directory would - // be a lie the host acts on. Named here one by one rather than waved through by - // pattern, so each new facility is a deliberate addition beside the issue that - // owns the vocabulary. - if host == "/var/run/docker.sock" { - continue - } - var covered bool - for d := range declared { - if host == d || strings.HasPrefix(host, strings.TrimRight(d, "/")+"/") { - covered = true - } - } - if !covered { - t.Errorf( - "%s: %v mounts %s and no resource declares it. The runtime will create it "+ - "as root, and the rule that keeps a directory holding data does not "+ - "reach a directory the mesh never declared", - name, r["id"], host) - } - } - } - } - if checked == 0 { - t.Fatal("no example mounts a host path, so this test proves nothing") - } -} - -func stringsOfTest(v any) []string { - list, ok := v.([]any) - if !ok { - return nil - } - out := make([]string, 0, len(list)) - for _, item := range list { - out = append(out, fmt.Sprint(item)) - } - return out -} - -// A resource uses only the keys its shape has. -// -// **The host is the only thing that knew, and it is five steps downstream.** A container carrying -// `restart-on` — which belongs to a service — composed into a declaration without complaint, was -// pushed, and was refused on the machine. The host refused *the whole declaration*, correctly, -// because applying the parts it understood would leave a machine that looks configured and is -// not. So one misplaced key stopped a module dead, and the only place that said so was a log on a -// lab machine after a seventeen-minute run. -// -// Nine of them had shipped across seven modules. -// -// The lists are written out rather than imported: the host is another repository and this is its -// wire format, like the shape of a grant file. Duplicated deliberately, and checked — a contract -// with two copies and no check is a contract until somebody edits one. -func TestAResourceUsesOnlyTheKeysItsShapeHas(t *testing.T) { - common := []string{"id", "type"} - shapes := map[string][]string{ - "file": {"path", "content", "bytes", "sealed", "secrets", "mode", "owner"}, - "directory": {"path", "mode", "owner"}, - "container": {"name", "image", "env", "env-file", "ports", "volumes", "args", "hosts", "network", "artifact"}, - "service": {"unit", "state", "boot", "restart-on"}, - "package": {"package", "state"}, - "network": {"name"}, - "archive": {"path", "artifact", "digest", "owner", "mode"}, - "user": {"name", "shell", "groups", "home"}, - "action": {"command", "verify", "in"}, - } - - found, _ := filepath.Glob("*.json") - var checked int - for _, name := range found { - for _, r := range read(t, name).Resources { - kind := fmt.Sprint(r["type"]) - allowed, known := shapes[kind] - if !known { - t.Errorf("%s: %v is a %q, which is not a shape the mesh has", name, r["id"], kind) - continue - } - for key := range r { - checked++ - // `merge` and `protected` are read by the control plane and removed before a - // machine sees them, so they are legal here and unknown to the host. - if key == "merge" || key == "protected" { - continue - } - if !slices.Contains(common, key) && !slices.Contains(allowed, key) { - t.Errorf( - "%s: %v is a %s and carries %q, which that shape does not have. It would "+ - "compose cleanly and be refused on the machine — and the host refuses "+ - "the whole declaration, so this stops the module entirely", - name, r["id"], kind, key) - } - } - } - } - if checked == 0 { - t.Fatal("no example declares a resource, so this test proves nothing") - } -} diff --git a/examples/modules/nextcloud.json b/examples/modules/nextcloud.json deleted file mode 100644 index 20faa8d..0000000 --- a/examples/modules/nextcloud.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "module": "nextcloud", - "version": "1", - "requires": [ - "postgres-database", - "s3-bucket" - ], - "contributes": { - "postgres-database": { - "name": "nextcloud" - }, - "s3-bucket": { - "bucket": "nextcloud" - } - }, - "binds": { - "postgres-database": "/var/lib/nextcloud-module/database.json", - "s3-bucket": "/var/lib/nextcloud-module/store.json" - }, - "secrets": { - "postgres-database": "/var/lib/nextcloud-module/database.secret", - "s3-bucket": "/var/lib/nextcloud-module/store.secret" - }, - "own-secrets": { - "admin": "/var/lib/nextcloud-module/admin.secret" - }, - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 80, - "protocol": "tcp", - "from": "mesh", - "why": "files and sync, over http; a public name is a route grant later" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/nextcloud-module", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/nextcloud-module/server.env", - "mode": "0600", - "content": "POSTGRES_HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nPOSTGRES_DB=nextcloud\nPOSTGRES_USER=${bound:postgres-database:as}\nPOSTGRES_PASSWORD=${secret:postgres-database}\nNEXTCLOUD_ADMIN_USER=admin\nNEXTCLOUD_ADMIN_PASSWORD=${secret:admin}\nOBJECTSTORE_S3_HOST=${bound:s3-bucket:at}\nOBJECTSTORE_S3_PORT=${bound:s3-bucket:port}\nOBJECTSTORE_S3_BUCKET=nextcloud\nOBJECTSTORE_S3_KEY=${bound:s3-bucket:as}\nOBJECTSTORE_S3_SECRET=${secret:s3-bucket}\nOBJECTSTORE_S3_SSL=false\nOBJECTSTORE_S3_USEPATH_STYLE=true\n" - }, - { - "id": "html", - "type": "directory", - "path": "/services/nextcloud/html", - "mode": "0750", - "owner": "33:33" - }, - { - "id": "server", - "type": "container", - "name": "nextcloud", - "image": "nextcloud@sha256:0b8261f6335af6b95264ce893b4d645857638e0fa151b5ba620f25f377318ae1", - "env-file": [ - "/var/lib/nextcloud-module/server.env" - ], - "ports": [ - "80" - ], - "volumes": [ - "/services/nextcloud/html:/var/www/html" - ] - } - ] -} diff --git a/examples/modules/nodered.json b/examples/modules/nodered.json deleted file mode 100644 index b5d90d9..0000000 --- a/examples/modules/nodered.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "module": "nodered", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 1880, - "protocol": "tcp", - "from": "mesh", - "why": "the flow editor and the endpoints flows expose" - } - ], - "resources": [ - { - "id": "data", - "type": "directory", - "path": "/services/nodered/data", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "nodered", - "image": "nodered/node-red@sha256:02a2b92a41b73d2bc388238b86e4fcaab7fb5466373adb24e1df6aa5845265ff", - "env": { - "TZ": "Etc/UTC" - }, - "ports": [ - "1880" - ], - "volumes": [ - "/services/nodered/data:/data" - ] - } - ] -} diff --git a/examples/modules/nzbget.json b/examples/modules/nzbget.json deleted file mode 100644 index 95b41ca..0000000 --- a/examples/modules/nzbget.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "module": "nzbget", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 6789, - "protocol": "tcp", - "from": "mesh", - "why": "the download client's pages" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/nzbget/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "media-downloads", - "type": "directory", - "path": "/services/media/downloads", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "nzbget", - "image": "lscr.io/linuxserver/nzbget@sha256:5f3d3fa71029004156eff2cbf4ef4455ce4ce59517cf13fa7d1d7c8a4cd2c8a4", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "6789" - ], - "volumes": [ - "/services/nzbget/config:/config", - "/services/media/downloads:/downloads" - ] - } - ] -} diff --git a/examples/modules/ombi.json b/examples/modules/ombi.json deleted file mode 100644 index 45d8a62..0000000 --- a/examples/modules/ombi.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "module": "ombi", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 3579, - "protocol": "tcp", - "from": "mesh", - "why": "requests from viewers" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/ombi/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "ombi", - "image": "lscr.io/linuxserver/ombi@sha256:a6f76ac521ba01eee2e9f0c23a3fed22e56630d97a04d5eeaeaa36c1e681640d", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "3579" - ], - "volumes": [ - "/services/ombi/config:/config" - ] - } - ] -} diff --git a/examples/modules/photos.json b/examples/modules/photos.json deleted file mode 100644 index e3bac87..0000000 --- a/examples/modules/photos.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "module": "photos", - "version": "1", - "requires": [ - "s3-bucket" - ], - "contributes": { - "s3-bucket": { - "bucket": "photos" - } - }, - "binds": { - "s3-bucket": "/etc/photos/store.json" - }, - "secrets": { - "s3-bucket": "/etc/photos/store.secret" - }, - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/etc/photos", - "mode": "0750" - }, - { - "id": "app", - "type": "container", - "name": "photos", - "image": "alpine@sha256:c64c687cbea9300178b30c95835354e34c4e4febc4badfe27102879de0483b5e", - "env": { - "PHOTOS_STORE": "/etc/photos/store.json", - "PHOTOS_STORE_SECRET_FILE": "/etc/photos/store.secret" - }, - "volumes": [ - "/etc/photos/store.json:/etc/photos/store.json:ro", - "/etc/photos/store.secret:/etc/photos/store.secret:ro" - ] - } - ] -} diff --git a/examples/modules/plex.json b/examples/modules/plex.json deleted file mode 100644 index d1287f3..0000000 --- a/examples/modules/plex.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "module": "plex", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 32400, - "protocol": "tcp", - "from": "mesh", - "why": "streaming and the app; reaching it from outside is a route grant later" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/plex/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "transcode", - "type": "directory", - "path": "/services/plex/transcode", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "media-movies", - "type": "directory", - "path": "/services/media/movies", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-series", - "type": "directory", - "path": "/services/media/series", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-anime", - "type": "directory", - "path": "/services/media/anime", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-music", - "type": "directory", - "path": "/services/media/music", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-audiobooks", - "type": "directory", - "path": "/services/media/audiobooks", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "plex", - "image": "plexinc/pms-docker@sha256:83a425ae9e133b1cb2cc3b809556e01c61cd8ff65c582e41b4374bc2210bac9e", - "network": "host", - "env": { - "PLEX_UID": "1000", - "PLEX_GID": "1000", - "TZ": "Etc/UTC" - }, - "volumes": [ - "/services/plex/config:/config", - "/services/plex/transcode:/transcode", - "/services/media/movies:/movies", - "/services/media/series:/series", - "/services/media/anime:/anime", - "/services/media/music:/music", - "/services/media/audiobooks:/audiobooks" - ] - } - ] -} diff --git a/examples/modules/portainer.json b/examples/modules/portainer.json deleted file mode 100644 index 0f0da7a..0000000 --- a/examples/modules/portainer.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "module": "portainer", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 9443, - "protocol": "tcp", - "from": "mesh", - "why": "the container dashboard, over its own tls" - } - ], - "resources": [ - { - "id": "data", - "type": "directory", - "path": "/services/portainer/data", - "mode": "0700" - }, - { - "id": "server", - "type": "container", - "name": "portainer", - "image": "portainer/portainer-ce@sha256:511f3f06c96fe3b993ebeaafde311c1959cae73a7ef825dba6397d51b450dffa", - "ports": [ - "9443" - ], - "volumes": [ - "/services/portainer/data:/data", - "/var/run/docker.sock:/var/run/docker.sock" - ] - } - ] -} diff --git a/examples/modules/postgres.json b/examples/modules/postgres.json deleted file mode 100644 index e2d8f03..0000000 --- a/examples/modules/postgres.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "module": "postgres", - "version": "1", - "provides": [ - { - "name": "postgres-database", - "scope": "mesh" - } - ], - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 5432, - "protocol": "tcp", - "from": "mesh", - "why": "modules on any machine that were granted a database" - } - ], - "serves": { - "postgres-database": {} - }, - "receives": { - "postgres-database": "/var/lib/postgres/grants/mesh.json" - }, - "grants": { - "postgres-database": "/var/lib/postgres/grants" - }, - "own-secrets": { - "superuser": "/var/lib/postgres/superuser.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/postgres", - "mode": "0700" - }, - { - "id": "grants", - "type": "directory", - "path": "/var/lib/postgres/grants", - "mode": "0700" - }, - { - "id": "superuser-env", - "type": "file", - "path": "/var/lib/postgres/superuser.env", - "mode": "0600", - "content": "POSTGRES_PASSWORD=${secret:superuser}\n" - }, - { - "id": "data", - "type": "directory", - "path": "/services/postgres/db-data", - "mode": "0700" - }, - { - "id": "net", - "type": "network", - "name": "postgres" - }, - { - "id": "server", - "type": "container", - "name": "postgres", - "image": "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee", - "network": "postgres", - "env": { - "POSTGRES_USER": "postgres", - "POSTGRES_DB": "postgres" - }, - "env-file": [ - "/var/lib/postgres/superuser.env" - ], - "ports": [ - "5432" - ], - "volumes": [ - "/services/postgres/db-data:/var/lib/postgresql/data" - ] - }, - { - "id": "provisioner", - "type": "container", - "name": "mesh-provision-postgres", - "image": "mesh-provision-postgres@sha256:0000000000000000000000000000000000000000000000000000000000000000", - "network": "postgres", - "env": { - "GRANTS": "/var/lib/postgres/grants", - "MESH_PROVISION_POSTGRES": "postgres://postgres@postgres:5432/postgres?sslmode=disable", - "MESH_PROVISION_PASSWORD_FILE": "/run/secrets/superuser" - }, - "volumes": [ - "/var/lib/postgres/grants:/var/lib/postgres/grants:ro", - "/var/lib/postgres/superuser.secret:/run/secrets/superuser:ro" - ] - } - ] -} diff --git a/examples/modules/qbittorrent.json b/examples/modules/qbittorrent.json deleted file mode 100644 index a8e8cb5..0000000 --- a/examples/modules/qbittorrent.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "module": "qbittorrent", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 8080, - "protocol": "tcp", - "from": "mesh", - "why": "the download client's pages" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/qbittorrent/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "media-downloads", - "type": "directory", - "path": "/services/media/downloads", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "qbittorrent", - "image": "lscr.io/linuxserver/qbittorrent@sha256:a00b6a597a3832a1814cde0ef60abc55c94644f3f80902c3432f6af6de8d4a96", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "8080" - ], - "volumes": [ - "/services/qbittorrent/config:/config", - "/services/media/downloads:/downloads" - ] - } - ] -} diff --git a/examples/modules/radarr.json b/examples/modules/radarr.json deleted file mode 100644 index 51a782f..0000000 --- a/examples/modules/radarr.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "module": "radarr", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 7878, - "protocol": "tcp", - "from": "mesh", - "why": "managing films" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/radarr/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "media-movies", - "type": "directory", - "path": "/services/media/movies", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-downloads", - "type": "directory", - "path": "/services/media/downloads", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "radarr", - "image": "lscr.io/linuxserver/radarr@sha256:119aaa4a4f7349bcd2a136c5373a0d7925b5479915c7dfe0c0ad352db2a6d438", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "7878" - ], - "volumes": [ - "/services/radarr/config:/config", - "/services/media/movies:/movies", - "/services/media/downloads:/downloads" - ] - } - ] -} diff --git a/examples/modules/redis.json b/examples/modules/redis.json deleted file mode 100644 index 5f223c6..0000000 --- a/examples/modules/redis.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "module": "redis", - "version": "1", - "provides": [ - { - "name": "redis-cache", - "scope": "mesh" - } - ], - "capabilities": [ - "container-runtime" - ], - "serves": { - "redis-cache": {} - }, - "receives": { - "redis-cache": "/var/lib/redis-module/grants/mesh.json" - }, - "grants": { - "redis-cache": "/var/lib/redis-module/grants" - }, - "own-secrets": { - "default": "/var/lib/redis-module/default.secret" - }, - "listens": [ - { - "port": 6379, - "protocol": "tcp", - "from": "mesh", - "why": "modules on any machine that were granted a cache" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/redis-module", - "mode": "0700" - }, - { - "id": "grants-dir", - "type": "directory", - "path": "/var/lib/redis-module/grants", - "mode": "0700" - }, - { - "id": "data", - "type": "directory", - "path": "/services/redis/data", - "mode": "0700", - "owner": "999:999" - }, - { - "id": "server-conf", - "type": "file", - "path": "/var/lib/redis-module/redis.conf", - "mode": "0600", - "content": "requirepass ${secret:default}\nappendonly yes\ndir /data\n", - "owner": "999:999" - }, - { - "id": "net", - "type": "network", - "name": "redis" - }, - { - "id": "server", - "type": "container", - "name": "redis", - "image": "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf", - "network": "redis", - "ports": [ - "6379" - ], - "volumes": [ - "/services/redis/data:/data", - "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro" - ], - "args": [ - "/etc/redis/redis.conf" - ] - }, - { - "id": "provisioner", - "type": "container", - "name": "mesh-provision-redis", - "image": "mesh-provision-redis@sha256:0000000000000000000000000000000000000000000000000000000000000000", - "network": "redis", - "env": { - "GRANTS": "/var/lib/redis-module/grants", - "MESH_PROVISION_REDIS": "redis:6379", - "MESH_PROVISION_PASSWORD_FILE": "/run/secrets/default" - }, - "volumes": [ - "/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro", - "/var/lib/redis-module/default.secret:/run/secrets/default:ro" - ] - } - ] -} diff --git a/examples/modules/resolv-conf.json b/examples/modules/resolv-conf.json deleted file mode 100644 index d7b161f..0000000 --- a/examples/modules/resolv-conf.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "module": "resolv-conf", - "version": "1", - - "requires": ["wildcard-resolution"], - "claims": [{"name": "the-resolver-configuration", "scope": "node"}], - - "resources": [ - {"id": "resolv", "type": "file", "path": "/etc/resolv.conf", "mode": "0644", - "content": "# Managed by the mesh.\n#\n# For a machine where nothing else owns this file. On one where systemd-resolved\n# or NetworkManager does, assign that module instead — this one and those claim\n# the same thing, so the mesh refuses the pair rather than letting them take\n# turns overwriting each other, which is the failure this claim exists to stop.\n#\n# The mesh's resolver first, because it answers only the mesh's names and\n# forwards nothing: a query it does not recognise falls through to the next\n# line rather than being answered wrongly.\nnameserver 127.0.0.55\n\n# And what this machine used before. Replace this line with the resolver this\n# machine should use for everything that is not the mesh — it is not the mesh's\n# to choose, and a public one written here by default would send every query\n# this machine makes somewhere nobody agreed to.\nnameserver 127.0.0.53\n"} - ] -} diff --git a/examples/modules/resolved-split-dns.json b/examples/modules/resolved-split-dns.json deleted file mode 100644 index dd3ea6d..0000000 --- a/examples/modules/resolved-split-dns.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "module": "resolved-split-dns", - "version": "1", - - "requires": ["wildcard-resolution"], - "claims": [{"name": "the-resolver-configuration", "scope": "node"}], - - "resources": [ - {"id": "drop-in", "type": "directory", "path": "/etc/systemd/resolved.conf.d", "mode": "0755"}, - - {"id": "route", "type": "file", - "path": "/etc/systemd/resolved.conf.d/mesh.conf", "mode": "0644", - "content": "# Managed by the mesh.\n#\n# **Only the mesh's names.** The tilde makes this a routing domain rather than a\n# search domain: queries under it go to the resolver below, and everything else\n# keeps going wherever this machine already sent it. A resolver that took over\n# all of DNS would be this module claiming the machine's whole network, which\n# is not what it says it claims.\n#\n# 127.0.0.55 is where the mesh's resolver answers on every machine — a fixed\n# address, so this file needs to know nothing about this particular machine.\n# systemd-resolved holds .53 and .54, which is why it is neither.\n[Resolve]\nDNS=127.0.0.55\nDomains=~internal\n"}, - - {"id": "resolved", "type": "service", "unit": "systemd-resolved.service", - "state": "running", "boot": "enabled", "restart-on": ["route"]} - ] -} diff --git a/examples/modules/searxng.json b/examples/modules/searxng.json deleted file mode 100644 index 8262a4b..0000000 --- a/examples/modules/searxng.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "module": "searxng", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "own-secrets": { - "secret": "/var/lib/searxng-module/secret.secret" - }, - "listens": [ - { - "port": 8080, - "protocol": "tcp", - "from": "mesh", - "why": "the search pages" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/searxng-module", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/searxng-module/server.env", - "mode": "0600", - "content": "SEARXNG_SECRET=${secret:secret}\nSEARXNG_VALKEY_URL=valkey://valkey:6379/0\n" - }, - { - "id": "net", - "type": "network", - "name": "searxng" - }, - { - "id": "cache", - "type": "container", - "name": "valkey", - "image": "valkey/valkey@sha256:b21fd94099dcd4bc6b2b9230daef69b6558b887ad4a2a1afe56ff6e745a88cdb", - "network": "searxng", - "args": [ - "valkey-server", - "--save", - "30", - "1", - "--loglevel", - "warning" - ], - "volumes": [ - "searxng-valkey-data:/data" - ] - }, - { - "id": "server", - "type": "container", - "name": "searxng", - "image": "searxng/searxng@sha256:c7cc75852051bf6254afda6ed1b920dd1677d8efe4ab141bf558f02e582f4371", - "network": "searxng", - "env-file": [ - "/var/lib/searxng-module/server.env" - ], - "ports": [ - "8080" - ] - } - ] -} diff --git a/examples/modules/sonarr.json b/examples/modules/sonarr.json deleted file mode 100644 index b42fb6c..0000000 --- a/examples/modules/sonarr.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "module": "sonarr", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 8989, - "protocol": "tcp", - "from": "mesh", - "why": "managing series" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/sonarr/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "media-series", - "type": "directory", - "path": "/services/media/series", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-anime", - "type": "directory", - "path": "/services/media/anime", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "media-downloads", - "type": "directory", - "path": "/services/media/downloads", - "mode": "0755", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "sonarr", - "image": "lscr.io/linuxserver/sonarr@sha256:c19aa4ecdf03d73e1d5c901da33744cb7eb4d921f89bafed1ca264601d7fa224", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "8989" - ], - "volumes": [ - "/services/sonarr/config:/config", - "/services/media/series:/series", - "/services/media/anime:/anime", - "/services/media/downloads:/downloads" - ] - } - ] -} diff --git a/examples/modules/tautulli.json b/examples/modules/tautulli.json deleted file mode 100644 index da05783..0000000 --- a/examples/modules/tautulli.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "module": "tautulli", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 8181, - "protocol": "tcp", - "from": "mesh", - "why": "watch statistics" - } - ], - "resources": [ - { - "id": "config", - "type": "directory", - "path": "/services/tautulli/config", - "mode": "0700", - "owner": "1000:1000" - }, - { - "id": "server", - "type": "container", - "name": "tautulli", - "image": "lscr.io/linuxserver/tautulli@sha256:13f03ecfc61a7af89d492677389771ac29682a72153ce08a5cd4faffbc0197e8", - "env": { - "PUID": "1000", - "PGID": "1000", - "TZ": "Etc/UTC" - }, - "ports": [ - "8181" - ], - "volumes": [ - "/services/tautulli/config:/config" - ] - } - ] -} diff --git a/examples/modules/umami.json b/examples/modules/umami.json deleted file mode 100644 index aa341e7..0000000 --- a/examples/modules/umami.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "module": "umami", - "version": "1", - "requires": [ - "postgres-database" - ], - "contributes": { - "postgres-database": { - "name": "umami" - } - }, - "binds": { - "postgres-database": "/var/lib/umami/database.json" - }, - "secrets": { - "postgres-database": "/var/lib/umami/database.secret" - }, - "own-secrets": { - "app-secret": "/var/lib/umami/app.secret" - }, - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 3000, - "protocol": "tcp", - "from": "mesh", - "why": "the analytics pages and the collection endpoint" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/umami", - "mode": "0700" - }, - { - "id": "server-env", - "type": "file", - "path": "/var/lib/umami/server.env", - "mode": "0600", - "content": "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/umami\nAPP_SECRET=${secret:app-secret}\n" - }, - { - "id": "server", - "type": "container", - "name": "umami", - "image": "ghcr.io/umami-software/umami@sha256:fa32d116cf20cad52cbc3fad9a63b46e7fa02299d8f967168eb453d49c476b4a", - "env-file": [ - "/var/lib/umami/server.env" - ], - "ports": [ - "3000" - ] - } - ] -} diff --git a/examples/modules/verdaccio.json b/examples/modules/verdaccio.json deleted file mode 100644 index 34196f2..0000000 --- a/examples/modules/verdaccio.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "module": "verdaccio", - "version": "1", - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 4873, - "protocol": "tcp", - "from": "mesh", - "why": "the package registry, for installs and publishes" - } - ], - "resources": [ - { - "id": "conf", - "type": "directory", - "path": "/services/verdaccio/conf", - "mode": "0755", - "owner": "10001:10001" - }, - { - "id": "storage", - "type": "directory", - "path": "/services/verdaccio/storage", - "mode": "0700", - "owner": "10001:10001" - }, - { - "id": "config", - "type": "file", - "path": "/services/verdaccio/conf/config.yaml", - "mode": "0644", - "content": "storage: /verdaccio/storage\nauth:\n htpasswd:\n file: /verdaccio/conf/htpasswd\n max_users: 10\nuplinks:\n npmjs:\n url: https://registry.npmjs.org/\npackages:\n \"**\":\n access: $all\n publish: $authenticated\n proxy: npmjs\nserver:\n keepAliveTimeout: 60\n maxBodySize: 10mb\nmiddlewares:\n audit:\n enabled: true\nlog:\n type: stdout\n format: pretty\n level: http\n" - }, - { - "id": "server", - "type": "container", - "name": "verdaccio", - "image": "verdaccio/verdaccio@sha256:fcb86134563534e2f634752e6c6c3edcdb78242ec16578c73ce39d1dadbaa801", - "ports": [ - "4873" - ], - "volumes": [ - "/services/verdaccio/storage:/verdaccio/storage", - "/services/verdaccio/conf:/verdaccio/conf" - ] - } - ] -} diff --git a/internal/catalogue/parseall_test.go b/internal/catalogue/parseall_test.go deleted file mode 100644 index 3885307..0000000 --- a/internal/catalogue/parseall_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package catalogue - -import ( - "os" - "path/filepath" - "testing" -) - -func TestEveryExampleManifestStillParses(t *testing.T) { - found, err := filepath.Glob("../../examples/modules/*.json") - if err != nil || len(found) == 0 { - t.Fatalf("no example manifests found: %v", err) - } - for _, f := range found { - raw, err := os.ReadFile(f) - if err != nil { - t.Fatal(err) - } - if _, err := ParseManifest(raw); err != nil { - t.Errorf("%s no longer parses: %v", filepath.Base(f), err) - } - } -} -- 2.54.0