Compare commits

..
2 Commits
Author SHA1 Message Date
jschoubben 3728b07d20 Leave gitea's 222:22 to its own decision
A controller test holds git-over-ssh at 222 as the mesh's fixed convention
when no node says otherwise (TestTheForgesSshPortIsTheMeshsFixedConvention
ByDefault): without a setting, the short form would publish 22:22 and
collide with the machine's own sshd. Changing that is a decision, not part
of this sweep.
2026-09-30 12:36:24 +02:00
jschoubben a49f598a4e Manifests publish software ports; the machine side is the assignment's
postgres, distribution, gitea, mailu, only-office and portainer published
long-form mappings (5432:5432, 222:22, 7080:80, 9090:9000 …): a machine port
in a definition (ADR 0038, the migration note). Each now publishes the
software's port and names it in listens (mailu's web/web-tls and
only-office's and portainer's web said the machine number); mailu's env
says ${port:80}/${port:443} instead of 7080/7443.

Nothing moves on novox. postgres, distribution and gitea already have their
machine ports as novox settings (6852, 5100, 222); mailu, only-office and
portainer need theirs set BEFORE this rolls out:
  settings set mailu {"ports":{"80":7080,"443":7443}} --node novox
  settings set only-office {"ports":{"80":9070}} --node novox
  settings set portainer {"ports":{"9000":9090,"9443":9443}} --node novox
Those pins are accepted by today's manifests too (GivenPorts answers to the
container port), so setting them first changes nothing either.

Verified with the controller's own publishedOn/portInto/GivenPorts on
novox's settings (+ the three pins): every container's published ports and
mailu's env file render byte-identical before and after — mesh-store
6852:5432, mesh-registry 5100:5000, gitea 222:22, mailu-front 7080:80
7443:443 (+ mail ports), only-office 9070:80, portainer 9090:9000 9443:9443.

Not included: nats publishes 127.0.0.1:8222:8222 — a loopback bind, which a
short form cannot express; its monitor port needs its own change.
2026-09-30 12:35:58 +02:00
125 changed files with 1125 additions and 5990 deletions
+11 -10
View File
@@ -9,13 +9,13 @@
"model-access"
],
"binds": {
"model-access": "${dir:state}/model.json"
"model-access": "/var/lib/anthropic-consumer/model.json"
},
"secrets": {
"model-access": "${dir:state}/access-token"
"model-access": "/var/lib/anthropic-consumer/access-token"
},
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/anthropic-consumer/broker"
},
"emits": [
"usage.session"
@@ -24,14 +24,14 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/anthropic-consumer",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/anthropic-consumer",
"mode": "0700"
},
{
"id": "claude-home",
@@ -42,6 +42,7 @@
{
"id": "out",
"type": "directory",
"path": "/var/lib/anthropic-consumer/out",
"mode": "0700"
},
{
@@ -55,7 +56,7 @@
"/app/modules/anthropic-consumer/dist/apply/index.js"
],
"volumes": [
"${dir:state}:/run/state"
"/var/lib/anthropic-consumer:/run/state"
],
"env": {
"MESH_MODEL_ACCESS_SECRET_FILE": "/run/state/access-token",
@@ -76,8 +77,8 @@
"/app/modules/anthropic-consumer/dist/usage/index.js"
],
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:state}:/run/state"
"/var/lib/mesh/anthropic-consumer/broker:/run/secrets/broker:ro",
"/var/lib/anthropic-consumer:/run/state"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
+8 -8
View File
@@ -9,13 +9,13 @@
"model-access"
],
"binds": {
"model-access": "${dir:mesh-state}/model.json"
"model-access": "/var/lib/mesh/anthropic-manager/model.json"
},
"secrets": {
"model-access": "${dir:mesh-state}/refresh-token"
"model-access": "/var/lib/mesh/anthropic-manager/refresh-token"
},
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/anthropic-manager/broker"
},
"emits": [
"usage.read"
@@ -24,13 +24,13 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/anthropic-manager",
"mode": "0700"
},
{
"id": "out",
"type": "directory",
"path": "${dir:mesh-state}/out",
"path": "/var/lib/mesh/anthropic-manager/out",
"mode": "0700"
},
{
@@ -45,8 +45,8 @@
"/app/modules/anthropic-manager/dist/refresh/index.js"
],
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}:/run/state"
"/var/lib/mesh/anthropic-manager/broker:/run/secrets/broker:ro",
"/var/lib/mesh/anthropic-manager:/run/state"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
+6 -5
View File
@@ -6,7 +6,7 @@
"**"
],
"own-secrets": {
"broker": "${dir:state}/broker"
"broker": "/var/lib/audit-logger/broker"
},
"build": {
"on": [
@@ -33,12 +33,13 @@
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/audit-logger",
"mode": "0700"
},
{
"id": "trail",
"type": "directory",
"path": "/var/lib/audit-logger/trail",
"mode": "0700"
},
{
@@ -47,8 +48,8 @@
"name": "mesh-audit-logger",
"network": "host",
"volumes": [
"${dir:state}/broker:/run/secrets/broker:ro",
"${dir:trail}:/trail"
"/var/lib/audit-logger/broker:/run/secrets/broker:ro",
"/var/lib/audit-logger/trail:/trail"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
+24 -55
View File
@@ -2,15 +2,12 @@
// module's tools and anything else baserow-specific import it; nothing outside baserow does.
//
// Baserow authenticates a person with email + password, exchanged for a JWT at /api/user/token-auth/.
// The standard image creates no admin from env, so the account is one a person made in Baserow: its
// password is the module's `admin` secret, accepted from the operator, and its email and the public
// host Baserow answers to reach the runtime config file the mesh mounts (the email from the
// assignment's settings). Until both are there fromEnv throws and the module exposes no tools — the
// same dormant-until-configured shape gitea uses for its token.
// Those credentials are the mesh's own: a person signs up in Baserow (the standard image creates no
// admin from env), and the credential is placed in the runtime config file the mesh mounts. Until
// that happens fromEnv throws and the module simply exposes no tools — the same dormant-until-
// configured shape gitea uses for its token.
import { readFileSync } from "node:fs";
import { request as httpRequest } from "node:http";
import { request as httpsRequest } from "node:https";
export interface BaserowApplication {
id: number;
@@ -71,63 +68,35 @@ export class BaserowClient {
return h;
}
/**
* One HTTP exchange. Not `fetch`: Node's fetch drops a caller's Host header and sends the URL's
* own, and Baserow answers only the host of its BASEROW_PUBLIC_URL — any other Host is looked up
* as a published builder site and gets 404, `/api/_health/` included. A co-located caller reaching
* it by container name must present the public host, so the request is made with node:http, which
* sends the Host it is given.
*/
private send(path: string, method: string, headers: Record<string, string>, body?: string): Promise<{ status: number; text: string }> {
const url = new URL(`${this.baseUrl}${path}`);
const request = url.protocol === "https:" ? httpsRequest : httpRequest;
// A length, never chunked: Baserow's server reads a chunked body as empty.
const sent = body === undefined ? headers : { ...headers, "Content-Length": String(Buffer.byteLength(body)) };
return new Promise((resolve, reject) => {
const req = request(url, { method, headers: sent }, (res) => {
let text = "";
res.setEncoding("utf8");
res.on("data", (chunk: string) => (text += chunk));
res.on("end", () => resolve({ status: res.statusCode ?? 0, text }));
res.on("error", reject);
});
req.on("error", reject);
if (body !== undefined) req.write(body);
req.end();
});
}
/** Exchange email + password for a JWT, caching it until Baserow refuses it. Handles both the
/** Exchange email + password for a JWT, caching it for the client's lifetime. Handles both the
* older `{ token }` and the newer `{ access_token }` response shapes. */
async authenticate(): Promise<string> {
if (this.token) return this.token;
const res = await this.send(
"/api/user/token-auth/",
"POST",
this.headers(),
JSON.stringify({ email: this.email, password: this.password }),
);
if (res.status < 200 || res.status >= 300) throw new Error(`baserow auth failed: ${res.status} ${res.text}`);
const data = JSON.parse(res.text) as { token?: string; access_token?: string };
const res = await fetch(`${this.baseUrl}/api/user/token-auth/`, {
method: "POST",
headers: this.headers(),
body: JSON.stringify({ email: this.email, password: this.password }),
});
if (!res.ok) throw new Error(`baserow auth failed: ${res.status} ${await res.text()}`);
const data = (await res.json()) as { token?: string; access_token?: string };
const token = data.access_token ?? data.token;
if (!token) throw new Error("baserow auth returned no token");
this.token = token;
return token;
}
/** An authenticated GET. A refused token is dropped and the call made once more with a fresh one:
* Baserow's access tokens expire after minutes, and the runtime lives for weeks. */
private async authed<T>(path: string): Promise<T> {
for (let attempt = 0; ; attempt++) {
const token = await this.authenticate();
const res = await this.send(path, "GET", this.headers({ Authorization: `JWT ${token}` }));
if (res.status === 401 && attempt === 0) {
this.token = null;
continue;
}
if (res.status < 200 || res.status >= 300) throw new Error(`baserow ${path}: ${res.status} ${res.text}`);
return (res.text ? JSON.parse(res.text) : null) as T;
}
private async authed<T>(path: string, options: RequestInit = {}): Promise<T> {
const token = await this.authenticate();
const res = await fetch(`${this.baseUrl}${path}`, {
...options,
headers: this.headers({
Authorization: `JWT ${token}`,
...(options.headers as Record<string, string> | undefined),
}),
});
if (!res.ok) throw new Error(`baserow ${path}: ${res.status} ${await res.text()}`);
const text = await res.text();
return (text ? JSON.parse(text) : null) as T;
}
/** The applications (databases) the account can see, across all its workspaces. */
+22 -21
View File
@@ -18,15 +18,15 @@
}
},
"binds": {
"postgres-database": "${dir:state}/database.json",
"route": "${dir:state}/route.json"
"postgres-database": "/var/lib/baserow/database.json",
"route": "/var/lib/baserow/route.json"
},
"secrets": {
"postgres-database": "${dir:state}/database.secret"
"postgres-database": "/var/lib/baserow/database.secret"
},
"own-secrets": {
"admin": "${dir:state}/admin.secret",
"broker": "${dir:mesh-state}/broker"
"secret-key": "/var/lib/baserow/secret-key.secret",
"broker": "/var/lib/mesh/baserow/broker"
},
"listens": [
{
@@ -34,34 +34,35 @@
"port": 80,
"protocol": "tcp",
"from": "mesh",
"why": "the Baserow web UI and REST API, served by the image's own Caddy; a public name is the route's"
"why": "the Baserow web UI and REST API; a public name is a route grant later"
}
],
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/baserow",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/baserow",
"mode": "0700"
},
{
"id": "data",
"type": "directory",
"path": "/services/baserow/data",
"mode": "0755",
"owner": "9999:9999"
},
{
"id": "server-env",
"type": "file",
"path": "${dir:state}/server.env",
"path": "/var/lib/baserow/server.env",
"mode": "0600",
"content": "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\nDATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\nDATABASE_PASSWORD_FILE=/run/secrets/database\nDISABLE_EMBEDDED_PSQL=true\nBASEROW_PUBLIC_URL=https://${bound:route:name}\n"
"content": "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\nDATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\nDATABASE_PASSWORD=${secret:postgres-database}\nSECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n"
},
{
"id": "net",
@@ -72,25 +73,25 @@
"id": "server",
"type": "container",
"name": "baserow",
"image": "baserow/baserow@sha256:263ea6c4b72c9eccabcd975ffe9fdebf23913a293a514bec6a3897a5e0a5a080",
"image": "baserow/baserow@sha256:834424a10413798567f76428f255dc259445b7f8dcec56598c05b4073bb2a124",
"network": "baserow",
"env-file": [
"${dir:state}/server.env"
"/var/lib/baserow/server.env"
],
"ports": [
"80"
],
"volumes": [
"${dir:data}:/baserow/data",
"${dir:state}/database.secret:/run/secrets/database:ro"
]
"/services/baserow/data:/baserow/data"
],
"secrets-in-environment": "baserow reads DATABASE_PASSWORD and SECRET_KEY with os.getenv and has no _FILE twin (settings/base.py); not convertible"
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/baserow/config.json",
"mode": "0600",
"content": "{\n \"password\": \"${secret:admin}\",\n \"host\": \"${bound:route:name}\"\n}\n",
"content": "{}\n",
"merge": "json"
},
{
@@ -99,8 +100,8 @@
"name": "mesh-baserow",
"network": "baserow",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro"
"/var/lib/mesh/baserow/broker:/run/secrets/broker:ro",
"/var/lib/mesh/baserow/config.json:/run/config/config.json:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
+13 -17
View File
@@ -8,8 +8,8 @@
"subtitle.downloaded"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker",
"api-key": "${dir:mesh-state}/api-key"
"broker": "/var/lib/mesh/bazarr/broker",
"api-key": "/var/lib/mesh/bazarr/api-key"
},
"listens": [
{
@@ -22,22 +22,18 @@
],
"accesses": [
{
"id": "movies",
"path": "/services/media/movies",
"mode": "read-write"
},
{
"id": "series",
"path": "/services/media/series",
"mode": "read-write"
},
{
"id": "anime",
"path": "/services/media/anime",
"mode": "read-write"
},
{
"id": "downloads",
"path": "/services/media/downloads",
"mode": "read"
}
@@ -46,8 +42,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/bazarr",
"mode": "0700"
},
{
"id": "config",
@@ -71,16 +67,16 @@
],
"volumes": [
"/services/bazarr/config:/config",
"${access:movies}:/movies",
"${access:series}:/series",
"${access:anime}:/anime",
"${access:downloads}:/downloads"
"/services/media/movies:/movies",
"/services/media/series:/series",
"/services/media/anime:/anime",
"/services/media/downloads:/downloads"
]
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/bazarr/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -91,9 +87,9 @@
"name": "mesh-bazarr",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/api-key:/run/secrets/api-key:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"/var/lib/mesh/bazarr/broker:/run/secrets/broker:ro",
"/var/lib/mesh/bazarr/api-key:/run/secrets/api-key:ro",
"/var/lib/mesh/bazarr/config.json:/run/config/config.json:ro",
"/services/bazarr/config:/var/lib/bazarr/config:ro"
],
"env": {
@@ -119,7 +115,7 @@
}
},
"binds": {
"route": "${dir:mesh-state}/route.json"
"route": "/var/lib/mesh/bazarr/route.json"
},
"build": {
"on": [
+7 -9
View File
@@ -11,7 +11,7 @@
],
"consumes": [],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/bookshelf/broker"
},
"listens": [
{
@@ -24,12 +24,10 @@
],
"accesses": [
{
"id": "books",
"path": "/services/media/books",
"mode": "read-write"
},
{
"id": "downloads",
"path": "/services/media/downloads",
"mode": "read-write"
}
@@ -38,8 +36,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/bookshelf",
"mode": "0700"
},
{
"id": "config",
@@ -63,8 +61,8 @@
],
"volumes": [
"/services/bookshelf/config:/config",
"${access:books}:/books",
"${access:downloads}:/downloads"
"/services/media/books:/books",
"/services/media/downloads:/downloads"
]
},
{
@@ -73,7 +71,7 @@
"name": "mesh-bookshelf",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"/var/lib/mesh/bookshelf/broker:/run/secrets/broker:ro",
"/services/bookshelf/config:/var/lib/bookshelf/config:ro"
],
"env": {
@@ -94,7 +92,7 @@
}
},
"binds": {
"route": "${dir:mesh-state}/route.json"
"route": "/var/lib/mesh/bookshelf/route.json"
},
"build": {
"on": [
+12 -12
View File
@@ -15,32 +15,33 @@
"npm-package-registry"
],
"binds": {
"npm-package-registry": "${dir:mesh-state}/package-registry.json"
"npm-package-registry": "/var/lib/mesh/builder/package-registry.json"
},
"secrets": {
"npm-package-registry": "${dir:mesh-state}/package-registry.secret"
"npm-package-registry": "/var/lib/mesh/builder/package-registry.secret"
},
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/builder/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/builder",
"mode": "0700"
},
{
"id": "workspace",
"type": "directory",
"path": "/var/lib/builder/workspace",
"mode": "0700"
},
{
"id": "builder-env",
"type": "file",
"path": "${dir:mesh-state}/builder.env",
"path": "/var/lib/mesh/builder/builder.env",
"mode": "0600",
"content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port}\nMESH_PACKAGE_BINDING=/run/mesh/package-registry.json\nMESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret\nMESH_WORKSPACE=${dir:workspace}\n"
"content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port}\nMESH_PACKAGE_BINDING=/run/mesh/package-registry.json\nMESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret\nMESH_WORKSPACE=/var/lib/builder/workspace\n"
},
{
"id": "server",
@@ -48,11 +49,11 @@
"name": "mesh-builder",
"artifact": "server",
"env-file": [
"${dir:mesh-state}/builder.env"
"/var/lib/mesh/builder/builder.env"
],
"volumes": [
"${dir:mesh-state}:/run/mesh:ro",
"${dir:workspace}:${dir:workspace}",
"/var/lib/mesh/builder:/run/mesh:ro",
"/var/lib/builder/workspace:/var/lib/builder/workspace",
"/var/run/docker.sock:/var/run/docker.sock"
],
"restart-on": [
@@ -68,8 +69,7 @@
"kind": "image",
"from": "Dockerfile",
"context": {
"seat": "git",
"repository": "novox/mesh-controller",
"repository": "https://git.novox.be/novox/mesh-controller.git",
"ref": "main"
}
}
+14 -13
View File
@@ -12,14 +12,14 @@
"public-dns": {}
},
"grants": {
"public-dns": "${dir:grants}"
"public-dns": "/var/lib/cloudflare-dns/grants"
},
"receives": {
"public-dns": "${dir:grants}/mesh.json"
"public-dns": "/var/lib/cloudflare-dns/grants/mesh.json"
},
"own-secrets": {
"token": "${dir:state}/token",
"broker": "${dir:mesh-state}/broker"
"token": "/var/lib/cloudflare-dns/token",
"broker": "/var/lib/mesh/cloudflare-dns/broker"
},
"emits": [
"record.created",
@@ -29,24 +29,25 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/cloudflare-dns",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/cloudflare-dns",
"mode": "0700"
},
{
"id": "grants",
"type": "directory",
"path": "/var/lib/cloudflare-dns/grants",
"mode": "0700"
},
{
"id": "config",
"type": "file",
"path": "${dir:state}/config.json",
"path": "/var/lib/cloudflare-dns/config.json",
"merge": "json",
"content": "{}",
"mode": "0600"
@@ -57,10 +58,10 @@
"name": "mesh-cloudflare-dns",
"network": "host",
"volumes": [
"${dir:state}/config.json:/run/config/config.json:ro",
"${dir:grants}:/grants",
"${dir:state}/token:/run/secrets/token:ro",
"${dir:mesh-state}/broker:/run/secrets/broker:ro"
"/var/lib/cloudflare-dns/config.json:/run/config/config.json:ro",
"/var/lib/cloudflare-dns/grants:/grants",
"/var/lib/cloudflare-dns/token:/run/secrets/token:ro",
"/var/lib/mesh/cloudflare-dns/broker:/run/secrets/broker:ro"
],
"env": {
"MESH_CLOUDFLARE_TOKEN_FILE": "/run/secrets/token",
+10 -10
View File
@@ -3,26 +3,26 @@
"version": "1",
"slug": "confl",
"own-secrets": {
"token": "${dir:state}/token",
"broker": "${dir:mesh-state}/broker"
"token": "/var/lib/confluence/token",
"broker": "/var/lib/mesh/confluence/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/confluence",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/confluence",
"mode": "0700"
},
{
"id": "config",
"type": "file",
"path": "${dir:state}/config.json",
"path": "/var/lib/confluence/config.json",
"merge": "json",
"content": "{}",
"mode": "0600"
@@ -33,9 +33,9 @@
"name": "mesh-runtime-confluence",
"network": "host",
"volumes": [
"${dir:state}/config.json:/run/config/config.json:ro",
"${dir:state}/token:/run/secrets/token:ro",
"${dir:mesh-state}/broker:/run/secrets/broker:ro"
"/var/lib/confluence/config.json:/run/config/config.json:ro",
"/var/lib/confluence/token:/run/secrets/token:ro",
"/var/lib/mesh/confluence/broker:/run/secrets/broker:ro"
],
"env": {
"MESH_CONFLUENCE_TOKEN_FILE": "/run/secrets/token",
+9 -12
View File
@@ -15,11 +15,11 @@
}
},
"binds": {
"route": "${dir:state}/route.json"
"route": "/var/lib/de-spiegel/route.json"
},
"own-secrets": {
"smtp-user": "${dir:state}/smtp-user.secret",
"smtp-pass": "${dir:state}/smtp-pass.secret"
"smtp-user": "/var/lib/de-spiegel/smtp-user.secret",
"smtp-pass": "/var/lib/de-spiegel/smtp-pass.secret"
},
"listens": [
{
@@ -27,20 +27,20 @@
"port": 35621,
"protocol": "tcp",
"from": "mesh",
"why": "the de-spiegel site and its /contact endpoint over http; its public name is a route grant, and route-proxy reaches it on this published port"
"why": "the de-spiegel site and its /contact endpoint over http; the public name de-spiegel.novox.be is a route grant, and route-proxy reaches it on this published port"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/de-spiegel",
"mode": "0700"
},
{
"id": "server-env",
"type": "file",
"path": "${dir:state}/server.env",
"path": "/var/lib/de-spiegel/server.env",
"mode": "0600",
"content": "SMTP_AUTH_USER=${secret:smtp-user}\nSMTP_AUTH_PASS=${secret:smtp-pass}\n"
},
@@ -56,15 +56,12 @@
"image": "registry-api.novox.be/novox/de-spiegel@sha256:e144b72ce9c145870470d765343549f2c60211728cd118b9ff0e4029f36342ba",
"network": "de-spiegel",
"env-file": [
"${dir:state}/server.env"
"/var/lib/de-spiegel/server.env"
],
"ports": [
"35621"
],
"secrets-in-environment": "the application's own code reads SMTP_AUTH_USER/PASS from the environment (de-spiegel server/index.js); converting is that repository's change",
"names-on-purpose": {
"registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)"
}
"secrets-in-environment": "the application's own code reads SMTP_AUTH_USER/PASS from the environment (de-spiegel server/index.js); converting is that repository's change"
}
]
}
+2 -2
View File
@@ -9,7 +9,7 @@
],
"claims": [
{
"name": "mesh-artifact-store",
"name": "the-artifact-store",
"scope": "mesh"
}
],
@@ -55,7 +55,7 @@
"name": "mesh-registry",
"image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373",
"ports": [
"5000:5000"
"5000"
],
"volumes": [
"/var/lib/mesh-registry:/var/lib/registry"
File diff suppressed because one or more lines are too long
-7
View File
@@ -95,13 +95,6 @@ export class GiteaClient {
if (res.status === 401) {
token = await this.tokens.renew(token);
res = await this.send(path, options, token);
} else if (res.status === 403) {
// A kept token minted before a scope was added lacks it. The forge says so; the source
// re-mints with the whole list and the call is retried once. Any other 403 stays a 403.
const text = await res.text();
if (!MintedToken.lacksScope(res.status, text)) throw new Error(`Gitea API ${path}: 403 ${text}`);
token = await this.tokens.renew(token);
res = await this.send(path, options, token);
}
if (!res.ok) throw new Error(`Gitea API ${path}: ${res.status} ${await res.text()}`);
if (res.status === 204) return null as T;
+8 -8
View File
@@ -86,19 +86,19 @@
}
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/gitea/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/gitea",
"mode": "0700"
},
{
"id": "runtime-state",
"type": "directory",
"path": "${dir:mesh-state}/state",
"path": "/var/lib/mesh/gitea/state",
"mode": "0700"
},
{
@@ -175,7 +175,7 @@
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/gitea/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -186,11 +186,11 @@
"name": "mesh-gitea",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"/var/lib/mesh/gitea/broker:/run/secrets/broker:ro",
"/var/lib/mesh/gitea/config.json:/run/config/config.json:ro",
"${dir:grants}:${dir:grants}:ro",
"${dir:state}/admin.secret:/run/secrets/admin:ro",
"${dir:runtime-state}:/run/state"
"/var/lib/mesh/gitea/state:/run/state"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
+2 -61
View File
@@ -29,7 +29,6 @@ interface Forge {
mints: number;
lastScopes: string[] | null;
tokens: Map<string, string>;
scopesOf: Map<string, string[]>;
admins: Map<string, string>;
close(): Promise<void>;
}
@@ -86,20 +85,6 @@ function fakeForge(): Promise<Forge> {
}
return json(res, 405, { message: "method not allowed" });
}
if (url.pathname === "/api/v1/repos/search") {
// The client lists through the search endpoint since 2026-09-28 (the forge's whole view);
// it sits under `repository`, which write:repository covers.
const h = req.headers.authorization ?? "";
const value = h.startsWith("token ") ? h.slice(6) : "";
if (![...forge.tokens.values()].includes(value)) return json(res, 401, { message: "token is required" });
if (!covers(forge.scopesOf.get(value) ?? [], "read:repository")) {
return json(res, 403, { message: `token does not have at least one of required scope(s), required=[read:repository]` });
}
return json(res, 200, {
ok: true,
data: [{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" }],
});
}
if (url.pathname === "/api/v1/user/repos") {
const h = req.headers.authorization ?? "";
const value = h.startsWith("token ") ? h.slice(6) : "";
@@ -116,21 +101,6 @@ function fakeForge(): Promise<Forge> {
{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" },
]);
}
const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/);
if (adminUser && req.method === "PATCH") {
const h = req.headers.authorization ?? "";
const value = h.startsWith("token ") ? h.slice(6) : "";
if (![...forge.tokens.values()].includes(value)) return json(res, 401, { message: "token is required" });
if (!covers(forge.scopesOf.get(value) ?? [], "write:admin")) {
return json(res, 403, {
message: `token does not have at least one of required scope(s), required=[write:admin]`,
});
}
const login = decodeURIComponent(adminUser[1]);
if (login === "untouchable") return json(res, 403, { message: "user untouchable may not be edited" });
const patch = await body(req);
return json(res, 200, { login, is_admin: patch?.admin === true });
}
return json(res, 404, { message: "no such route in the fake" });
});
return new Promise((resolve) => {
@@ -141,7 +111,6 @@ function fakeForge(): Promise<Forge> {
get mints() { return forge.mints; },
get lastScopes() { return forge.lastScopes; },
tokens: forge.tokens,
scopesOf: forge.scopesOf,
admins: forge.admins,
close: () => new Promise((r) => server.close(() => r())),
});
@@ -183,14 +152,14 @@ function minted(env: NodeJS.ProcessEnv, logs: string[]): GiteaClient {
const forge = await fakeForge();
after(() => forge.close());
test("first start: mints with the admin account, keeps the token at 0600, asks for the tools' scopes only", async () => {
test("first start: mints with the admin account, keeps the token at 0600, asks for two scopes only", async () => {
const { env, file, logs } = await delivered(forge);
const repos = await minted(env, logs).listRepos();
assert.equal(repos[0]?.full_name, "novox/hq");
assert.equal(forge.mints, 1);
assert.deepEqual(forge.lastScopes, ["write:repository", "write:issue", "read:user", "write:admin"]);
assert.deepEqual(forge.lastScopes, ["write:repository", "write:issue", "read:user"]);
assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]);
const token = forge.tokens.get("mesh-tools")!;
assert.equal(await readFile(file, "utf8"), token + "\n");
@@ -228,34 +197,6 @@ test("the forge rejects the kept token (its data was restored): minted afresh, o
assert.ok(logs.some((l) => l.startsWith("the forge rejected the kept token")), logs.join("\n"));
});
test("a kept token from before write:admin: the forge refuses the admin route for the scope, the token is re-minted with the whole list, and the call goes through", async () => {
const { env, file, logs } = await delivered(forge);
const client = minted(env, logs);
await client.listRepos();
const before = forge.mints;
const old = forge.tokens.get("mesh-tools")!;
forge.scopesOf.set(old, ["write:repository", "write:issue", "read:user"]); // minted by the previous build
const user = await client.api<{ login: string; is_admin: boolean }>("/admin/users/mesh_novox_builder", {
method: "PATCH",
body: JSON.stringify({ admin: true }),
});
assert.equal(user.is_admin, true);
assert.equal(forge.mints, before + 1);
assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]);
assert.notEqual(forge.tokens.get("mesh-tools"), old);
assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n");
assert.ok(logs.some((l) => l.startsWith("the forge rejected the kept token")), logs.join("\n"));
// A 403 that is not about scopes is the forge's answer, not a reason to mint.
const again = forge.mints;
await assert.rejects(
client.api("/admin/users/untouchable", { method: "PATCH", body: JSON.stringify({ admin: true }) }),
/403 .*untouchable/,
);
assert.equal(forge.mints, again);
});
test("the kept file is gone but the forge still holds a token by that name: replaced, not refused", async () => {
const { env, file, logs } = await delivered(forge);
await minted(env, logs).listRepos();
+3 -14
View File
@@ -34,21 +34,15 @@ export const TOKEN_NAME = "mesh-tools";
* It sits under the `user` category despite listing repositories, not `repository`
* — confirmed against the running forge (1.27.3), which answered
* `required=[read:user]` to a token carrying only the other two.
* write:admin — /admin/users: the forge's own users are the mesh's to settle, such as making
* the builder's login a site admin so every repository the mesh may build is
* clonable (novox/hq 229). Nothing under /orgs or write:user.
*
* A token kept from before a scope was added lacks it: the forge answers such a call with
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
* 401 — the source re-mints by name, with the whole list, and the call is retried once.
* Nothing under /admin, /orgs or write:user — the escape-hatch tool reaches only what these three cover.
*/
export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user", "write:admin"];
export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user"];
/** Where a client's token comes from, and what to do when the forge says it is wrong. */
export interface TokenSource {
/** The token to authenticate with now; minted, read or configured. */
current(): Promise<string>;
/** The forge answered 401 to `rejected`, or 403 for a scope it lacks. A fresh token, or a plain error when there is nothing to renew with. */
/** The forge answered 401 to `rejected`. A fresh token, or a plain error when there is nothing to renew with. */
renew(rejected: string): Promise<string>;
}
@@ -176,11 +170,6 @@ export class MintedToken implements TokenSource {
return this.mint("the forge rejected the kept token — minting a fresh one");
}
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
static lacksScope(status: number, body: string): boolean {
return status === 403 && /required scope/i.test(body);
}
/** One mint at a time: concurrent first calls share it, rather than each minting its own. */
private mint(why: string): Promise<string> {
if (this.inflight === null) {
+10 -10
View File
@@ -2,26 +2,26 @@
"module": "gitlab",
"version": "1",
"own-secrets": {
"token": "${dir:state}/token",
"broker": "${dir:mesh-state}/broker"
"token": "/var/lib/gitlab/token",
"broker": "/var/lib/mesh/gitlab/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/gitlab",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/gitlab",
"mode": "0700"
},
{
"id": "config",
"type": "file",
"path": "${dir:state}/config.json",
"path": "/var/lib/gitlab/config.json",
"merge": "json",
"content": "{}",
"mode": "0600"
@@ -32,9 +32,9 @@
"name": "mesh-runtime-gitlab",
"network": "host",
"volumes": [
"${dir:state}/config.json:/run/config/config.json:ro",
"${dir:state}/token:/run/secrets/token:ro",
"${dir:mesh-state}/broker:/run/secrets/broker:ro"
"/var/lib/gitlab/config.json:/run/config/config.json:ro",
"/var/lib/gitlab/token:/run/secrets/token:ro",
"/var/lib/mesh/gitlab/broker:/run/secrets/broker:ro"
],
"env": {
"MESH_GITLAB_TOKEN_FILE": "/run/secrets/token",
+22 -80
View File
@@ -5,8 +5,8 @@
"alert.firing"
],
"own-secrets": {
"admin": "${dir:mesh-state}/admin",
"broker": "${dir:mesh-state}/broker"
"admin": "/var/lib/grafana-module/admin.secret",
"broker": "/var/lib/mesh/grafana/broker"
},
"capabilities": [
"container-runtime"
@@ -24,93 +24,51 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/grafana",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/grafana-module",
"mode": "0700"
},
{
"id": "data",
"type": "directory",
"path": "/services/grafana/data",
"mode": "0700",
"owner": "472:472"
},
{
"id": "admin-secret",
"id": "server-env",
"type": "file",
"path": "${dir:state}/admin.secret",
"mode": "0400",
"owner": "472:472",
"content": "${secret:admin}"
},
{
"id": "oidc-secret",
"type": "file",
"path": "${dir:state}/oidc-client.secret",
"mode": "0400",
"owner": "472:472",
"content": "${secret:oidc-client}"
},
{
"id": "oidc-env",
"type": "file",
"path": "${dir:state}/oidc.env",
"mode": "0644",
"content": "GF_SERVER_ROOT_URL=https://${bound:route:name}\nGF_AUTH_GENERIC_OAUTH_ENABLED=true\nGF_AUTH_GENERIC_OAUTH_NAME=Keycloak\nGF_AUTH_GENERIC_OAUTH_CLIENT_ID=${bound:oidc-client:as}\nGF_AUTH_GENERIC_OAUTH_CLIENT_SECRET__FILE=/run/secrets/oidc-client\nGF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile roles\nGF_AUTH_GENERIC_OAUTH_AUTH_URL=${bound:oidc-client:issuer}${bound:oidc-client:authorization-path}\nGF_AUTH_GENERIC_OAUTH_TOKEN_URL=${bound:oidc-client:issuer}${bound:oidc-client:token-path}\nGF_AUTH_GENERIC_OAUTH_API_URL=${bound:oidc-client:issuer}${bound:oidc-client:userinfo-path}\nGF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'admin') && 'Admin' || contains(realm_access.roles[*], 'admin') && 'Admin' || 'Viewer'\nGF_AUTH_GENERIC_OAUTH_USE_PKCE=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true\n"
},
{
"id": "influxdb-secret",
"type": "file",
"path": "${dir:state}/influxdb-api.secret",
"mode": "0400",
"owner": "472:472",
"content": "${secret:influxdb-api}"
},
{
"id": "influxdb-datasource",
"type": "file",
"path": "${dir:state}/datasource-influxdb.yaml",
"mode": "0644",
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding; grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb-api}\n"
"path": "/var/lib/grafana-module/server.env",
"mode": "0600",
"content": "GF_SECURITY_ADMIN_PASSWORD=${secret:admin}\n"
},
{
"id": "server",
"type": "container",
"name": "grafana",
"image": "grafana/grafana@sha256:ac461fb352abc50da10a51c7d02462e9c05488f11f53f14b3ad79a8145f638a0",
"image": "grafana/grafana@sha256:f772d434e8fab0049deb2b1b30abd43342bcfca1537614aa8d36080232cf4283",
"ports": [
"3000"
],
"volumes": [
"${dir:data}:/var/lib/grafana",
"${dir:state}/admin.secret:/run/secrets/admin:ro",
"${dir:state}/oidc-client.secret:/run/secrets/oidc-client:ro",
"${dir:state}/influxdb-api.secret:/run/secrets/influxdb-api:ro",
"${dir:state}/datasource-influxdb.yaml:/etc/grafana/provisioning/datasources/mesh-influxdb.yaml:ro"
"/services/grafana/data:/var/lib/grafana"
],
"env": {
"GF_SECURITY_ADMIN_PASSWORD__FILE": "/run/secrets/admin"
},
"env-file": [
"${dir:state}/oidc.env"
"/var/lib/grafana-module/server.env"
],
"restart-on": [
"oidc-env",
"oidc-secret",
"influxdb-datasource",
"influxdb-secret"
]
"secrets-in-environment": "grafana honours GF_SECURITY_ADMIN_PASSWORD__FILE; convertible, awaiting a bed that exercises the admin password (assigned-grafana serves tools only)"
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/grafana/config.json",
"mode": "0600",
"content": "{\n \"user\": \"admin\",\n \"password\": \"${secret:admin}\"\n}\n",
"content": "{}\n",
"merge": "json"
},
{
@@ -119,12 +77,12 @@
"name": "mesh-grafana",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro"
"/var/lib/mesh/grafana/broker:/run/secrets/broker:ro",
"/var/lib/mesh/grafana/config.json:/run/config/config.json:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_GRAFANA_URL": "http://127.0.0.1:${port:3000}",
"MESH_GRAFANA_URL": "http://127.0.0.1:3000",
"MESH_GRAFANA_CONFIG_FILE": "/run/config/config.json"
},
"restart-on": [
@@ -134,32 +92,16 @@
}
],
"requires": [
"route",
"oidc-client",
"influxdb-api"
"route"
],
"contributes": {
"route": {
"label": "grafana",
"endpoint": "web"
},
"oidc-client": {
"label": "grafana",
"endpoint": "web",
"callback": "/login/generic_oauth"
},
"influxdb-api": {
"access": "read"
}
},
"binds": {
"route": "${dir:state}/route.json",
"oidc-client": "${dir:state}/oidc.json",
"influxdb-api": "${dir:state}/influxdb.json"
},
"secrets": {
"oidc-client": "${dir:mesh-state}/oidc-client",
"influxdb-api": "${dir:mesh-state}/influxdb-api"
"route": "/var/lib/mesh/grafana/route.json"
},
"build": {
"on": [
+5 -5
View File
@@ -15,7 +15,7 @@
}
},
"binds": {
"route": "${dir:state}/route.json"
"route": "/var/lib/hello-web/route.json"
},
"listens": [
{
@@ -30,13 +30,13 @@
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/hello-web",
"mode": "0700"
},
{
"id": "page",
"type": "file",
"path": "${dir:state}/index.html",
"path": "/var/lib/hello-web/index.html",
"mode": "0644",
"content": "hello from hello-web, routed by the mesh\n"
},
@@ -54,7 +54,7 @@
"8080"
],
"volumes": [
"${dir:state}/index.html:/www/index.html:ro"
"/var/lib/hello-web/index.html:/www/index.html:ro"
],
"args": [
"sh",
+1 -4
View File
@@ -13,7 +13,7 @@ ARG RUNTIME_BASE
FROM ${BUILD_BASE} AS build
WORKDIR /app/modules/home-assistant
COPY . .
RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts tools/index.ts provisions/hass.ts provisions/probe.ts provisions/connections.ts provisions/mesh.ts provisions/index.ts \
RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts tools/index.ts \
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
FROM ${RUNTIME_BASE}
@@ -22,6 +22,3 @@ COPY --from=build /app/modules/home-assistant/dist /app/modules/home-assistant/d
# provider's provisioner runs its reconcile loop in the same process, with the broker connected —
# the convention novox/hq issues 060/061 settled.
ENV MESH_TOOL_MODULES=/app/modules/home-assistant/dist/index.js,/app/modules/home-assistant/dist/tools/index.js
# NOT dist/provisions/index.js: that is a step the host runs to completion, named by the
# `provisions` container's args as `mesh-tools run …` (novox/hq ADR 0052). Listed here it would run
# inside the serving sidecar too, and exit it.
+20 -100
View File
@@ -9,8 +9,8 @@
"state.changed"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker",
"token": "${dir:mesh-state}/token"
"broker": "/var/lib/mesh/home-assistant/broker",
"token": "/var/lib/mesh/home-assistant/token"
},
"listens": [
{
@@ -18,63 +18,40 @@
"port": 8123,
"protocol": "tcp",
"from": "mesh",
"why": "the dashboard, the API and the companion apps"
},
{
"name": "sonos-events",
"port": 1400,
"protocol": "tcp",
"from": "mesh",
"why": "the Sonos integration's event callback: speakers push their state changes here"
},
{
"name": "webrtc",
"port": 18555,
"protocol": "tcp",
"from": "mesh",
"why": "the bundled go2rtc's WebRTC port, which camera streams to a browser use"
"why": "the dashboard and the API"
}
],
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/mesh/home-assistant",
"mode": "0700"
},
{
"id": "config",
"type": "directory",
"mode": "0700"
},
{
"id": "written",
"type": "directory",
"mode": "0700"
"path": "/services/home-assistant/config",
"mode": "0700",
"owner": "1000:1000"
},
{
"id": "server",
"type": "container",
"name": "home-assistant",
"image": "ghcr.io/home-assistant/home-assistant@sha256:d8922685169707fd91e8b9729902d975f06157d005e422874d201e0261dda196",
"image": "ghcr.io/home-assistant/home-assistant@sha256:14931c6b13756317849f46da1d01b45937a1150db66c081cfe529d48215943fe",
"network": "host",
"env": {
"TZ": "Etc/UTC"
},
"volumes": [
"${dir:config}:/config"
"/services/home-assistant/config:/config"
]
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/home-assistant/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -85,92 +62,35 @@
"name": "mesh-home-assistant",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/token:/run/secrets/token:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro"
"/var/lib/mesh/home-assistant/broker:/run/secrets/broker:ro",
"/var/lib/mesh/home-assistant/token:/run/secrets/token:ro",
"/var/lib/mesh/home-assistant/config.json:/run/config/config.json:ro",
"/services/home-assistant/config:/var/lib/home-assistant/config:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_HOMEASSISTANT_URL": "http://127.0.0.1:${port:8123}",
"MESH_HOMEASSISTANT_URL": "http://127.0.0.1:8123",
"MESH_HOMEASSISTANT_TOKEN_FILE": "/run/secrets/token",
"MESH_HOMEASSISTANT_CONFIG_FILE": "/run/config/config.json"
"MESH_HOMEASSISTANT_CONFIG_FILE": "/run/config/config.json",
"MESH_HOMEASSISTANT_CONFIG_DIR": "/var/lib/home-assistant/config"
},
"restart-on": [
"runtime-config"
],
"artifact": "runtime"
},
{
"id": "provisions",
"type": "container",
"name": "mesh-home-assistant-provisions",
"network": "host",
"run-once": true,
"volumes": [
"${dir:mesh-state}/token:/run/secrets/token:ro",
"${dir:written}:/var/lib/home-assistant-provisions",
"${dir:state}/mqtt-topic.json:/run/provisions/mqtt-topic.json:ro",
"${dir:state}/mqtt-topic.secret:/run/provisions/mqtt-topic.secret:ro",
"${dir:state}/sonarr-api.json:/run/provisions/sonarr-api.json:ro",
"${dir:state}/sonarr-api.secret:/run/provisions/sonarr-api.secret:ro",
"${dir:state}/radarr-api.json:/run/provisions/radarr-api.json:ro",
"${dir:state}/radarr-api.secret:/run/provisions/radarr-api.secret:ro",
"${dir:state}/lidarr-api.json:/run/provisions/lidarr-api.json:ro",
"${dir:state}/lidarr-api.secret:/run/provisions/lidarr-api.secret:ro"
],
"env": {
"MESH_HOMEASSISTANT_URL": "http://127.0.0.1:${port:8123}",
"MESH_HOMEASSISTANT_TOKEN_FILE": "/run/secrets/token",
"MESH_PROVISIONS_DIR": "/run/provisions",
"MESH_WRITTEN_DIR": "/var/lib/home-assistant-provisions"
},
"args": [
"run",
"/app/modules/home-assistant/dist/provisions/index.js"
],
"restart-on": [
"bound-mqtt-topic",
"secret-mqtt-topic",
"bound-sonarr-api",
"secret-sonarr-api",
"bound-radarr-api",
"secret-radarr-api",
"bound-lidarr-api",
"secret-lidarr-api"
],
"artifact": "runtime"
}
],
"requires": [
"lidarr-api",
"mqtt-topic",
"radarr-api",
"route",
"sonarr-api"
"route"
],
"contributes": {
"mqtt-topic": {
"topics": [
"#"
]
},
"route": {
"label": "home-assistant",
"endpoint": "web"
}
},
"binds": {
"route": "${dir:state}/route.json",
"mqtt-topic": "${dir:state}/mqtt-topic.json",
"sonarr-api": "${dir:state}/sonarr-api.json",
"radarr-api": "${dir:state}/radarr-api.json",
"lidarr-api": "${dir:state}/lidarr-api.json"
},
"secrets": {
"mqtt-topic": "${dir:state}/mqtt-topic.secret",
"sonarr-api": "${dir:state}/sonarr-api.secret",
"radarr-api": "${dir:state}/radarr-api.secret",
"lidarr-api": "${dir:state}/lidarr-api.secret"
"route": "/var/lib/mesh/home-assistant/route.json"
},
"build": {
"on": [
+1 -6
View File
@@ -1,14 +1,9 @@
{
"name": "@novox/module-home-assistant",
"version": "0.1.0",
"description": "home-assistant \u2014 home automation platform. Its API client, tools and events live here (novox/hq ADR 0039).",
"description": "home-assistant — home automation platform. Its API client, tools and events live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"scripts": {
"build": "tsc client.ts index.ts tools/index.ts provisions/hass.ts provisions/probe.ts provisions/connections.ts provisions/mesh.ts provisions/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
"typecheck": "tsc -p tsconfig.json",
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
},
"dependencies": {
"@novox/mesh-sdk": "^0.1.0"
},
@@ -1,486 +0,0 @@
// How home-assistant's provisions step brings Home Assistant's integrations in line with what the
// mesh bound: the MQTT integration to `mqtt-topic`, the Sonarr, Radarr and Lidarr integrations to
// `sonarr-api`, `radarr-api` and `lidarr-api`. Pure logic over two seams — Home Assistant's config
// flows (hass.ts) and the broker/apps — so it is tested against fakes (test/provisions.test.ts).
//
// The half that reads files and talks HTTP lives beside it (mesh.ts, hass.ts, probe.ts, index.ts).
import { createHash } from "node:crypto";
import type { Hass, SchemaField } from "./hass.js";
import type { Probe } from "./probe.js";
/** What the mesh wrote at `binds.<provision>` (the controller's binding document). */
export interface Binding {
provision?: string;
from?: string;
at?: string;
as?: string;
serves?: Record<string, unknown>;
}
/** How one provision came out. Never carries a credential. */
export type Outcome =
| { what: string; result: "unchanged"; note?: string }
| { what: string; result: "written"; fields: string[]; note?: string }
| { what: string; result: "equivalent"; note: string }
| { what: string; result: "refused"; problem: string };
/** A port the binding serves, or undefined when it names none usable. */
export function portOf(serves: Record<string, unknown> | undefined): number | undefined {
const port = Number(serves?.port);
return Number.isInteger(port) && port > 0 && port <= 65535 ? port : undefined;
}
/** A host as it goes into a URL: an IPv6 literal bracketed. */
export function urlHost(host: string): string {
return host.includes(":") && !host.startsWith("[") ? `[${host}]` : host;
}
/**
* What this step last wrote, per target, as a digest: the only way to know "already as the mesh
* says" for a credential Home Assistant will not show back. A sha256 over the target and the values,
* never the values; kept in the module's own placed directory.
*/
export interface Marks {
get(name: string): Promise<string | undefined>;
set(name: string, digest: string): Promise<void>;
}
export function digest(...parts: (string | number)[]): string {
return createHash("sha256").update(parts.map(String).join("\u0000")).digest("hex");
}
/** An error as text with the credential taken out, raw and URL-encoded. */
export function scrub(err: unknown, ...secrets: (string | undefined)[]): string {
let text = err instanceof Error ? err.message : String(err);
for (const s of secrets) {
if (!s) continue;
for (const form of new Set([s, encodeURIComponent(s)])) text = text.split(form).join("***");
}
return text;
}
/**
* What a form would submit if a person pressed "submit" without touching it: each field's
* suggested value (what Home Assistant pre-fills from the entry), else its default; a section's
* fields nested under its name. The step lays only the connection fields over this, so every other
* choice the entry carries is sent back exactly as Home Assistant showed it.
*/
export function formValues(schema: readonly SchemaField[] | null | undefined): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const field of schema ?? []) {
if (Array.isArray(field.schema)) {
out[field.name] = formValues(field.schema);
continue;
}
const suggested = field.description?.suggested_value;
if (suggested !== undefined && suggested !== null) out[field.name] = suggested;
else if (field.default !== undefined) out[field.name] = field.default;
}
return out;
}
/** Whether a form has a field of this name at its top level. */
export function hasField(schema: readonly SchemaField[] | null | undefined, name: string): boolean {
return (schema ?? []).some((f) => f.name === name);
}
// ---- MQTT ----
// Home Assistant's MQTT integration, pointed at the broker the mesh bound — `mqtt-topic`.
//
// **Why a step.** Home Assistant keeps its broker, login and password in its MQTT config entry
// (`.storage/core.config_entries`), not in a file the mesh could fill with `${bound:mqtt-topic:at}`.
// So this reads the binding and the pair credential and makes the entry say the same thing, through
// the MQTT integration's own reconfigure flow — the flow its "Reconfigure" button runs, which tests
// the connection itself and saves nothing it could not connect with.
//
// **Only the connection, and only when it differs.** Broker, port, username, password. The protocol
// version, client id, keepalive, TLS choices and discovery options the entry holds are sent back
// exactly as Home Assistant pre-filled them. Whether the password already matches cannot be read
// back (Home Assistant never shows a stored password), so the step keeps a digest of what it last
// wrote: equal broker/port/username and an equal digest is "already as the mesh says".
//
// **Nothing loses its connection without someone seeing it.** Before Home Assistant is touched the
// broker itself is asked whether it takes the delivered login (the provisioner creates it within
// seconds of the grant): if not, nothing is written and the step fails saying why, and Home
// Assistant keeps the login it has — the carried `luffy` on ace, which mosquitto keeps. If Home
// Assistant's own connection test refuses the new settings, the flow saves nothing, and the step
// fails with Home Assistant's reason. A login that may not subscribe to the discovery topics is said
// as a warning: discovery would find nothing.
export const MQTT_PROVISION = "mqtt-topic";
/** Home Assistant's discovery prefix, subscribed to whenever discovery is on (the default). */
export const DISCOVERY_FILTER = "homeassistant/#";
export interface MqttWanted {
host: string;
port: number;
username: string;
password: string;
}
export type Wanted = { ok: true; want: MqttWanted } | { ok: false; problem: string };
/** The broker, port and login the mesh says Home Assistant uses. */
export function wantedMqtt(binding: Binding | undefined, credential: string | undefined): Wanted {
if (!binding) return { ok: false, problem: `no binding for ${MQTT_PROVISION} was delivered — the mesh writes it before this step runs` };
const host = typeof binding.at === "string" ? binding.at.trim() : "";
if (!host) return { ok: false, problem: `the ${MQTT_PROVISION} binding names no host (at)` };
const port = portOf(binding.serves);
if (port === undefined) return { ok: false, problem: `the ${MQTT_PROVISION} binding serves no usable port (${String(binding.serves?.port)})` };
const scheme = binding.serves?.scheme;
if (scheme !== undefined && scheme !== "mqtt") {
return { ok: false, problem: `the ${MQTT_PROVISION} binding serves scheme ${String(scheme)}; this step writes plain MQTT` };
}
const username = typeof binding.as === "string" ? binding.as.trim() : "";
if (!username) return { ok: false, problem: `the ${MQTT_PROVISION} binding names no login (as)` };
const password = (credential ?? "").replace(/\n$/, "");
if (!password) return { ok: false, problem: `the ${MQTT_PROVISION} credential is empty or was not delivered` };
return { ok: true, want: { host, port, username, password } };
}
export interface MqttDeps {
hass: Hass;
probe: Probe;
marks: Marks;
}
const markFor = (entryId: string, w: MqttWanted): string => digest("mqtt", entryId, w.host, w.port, w.username, w.password);
/** Bring Home Assistant's MQTT entry in line with the mesh. Never throws: every failure is an outcome. */
export async function reconcileMqtt(deps: MqttDeps, binding: Binding | undefined, credential: string | undefined): Promise<Outcome> {
const what = "mqtt";
const w = wantedMqtt(binding, credential);
if ("problem" in w) return { what, result: "refused", problem: w.problem };
const want = w.want;
// The broker first: a login it does not take is never written into Home Assistant.
let note: string | undefined;
try {
const probe = await deps.probe(want.host, want.port, want.username, want.password, DISCOVERY_FILTER);
if (probe.connack === 4 || probe.connack === 5) {
return {
what,
result: "refused",
problem:
`the broker at ${want.host}:${want.port} does not (yet) take the login ${want.username} with the delivered ` +
`password (CONNACK ${probe.connack}); mosquitto's provisioner creates it from the grant — nothing was ` +
`written, and Home Assistant keeps the broker login it has`,
};
}
if (probe.connack !== 0) {
return { what, result: "refused", problem: `the broker at ${want.host}:${want.port} answered CONNACK ${probe.connack}; nothing was written` };
}
if (probe.suback === 0x80) {
note =
`warning: ${want.username} may not subscribe to ${DISCOVERY_FILTER} — MQTT discovery will find nothing; ` +
`grant it with the mqtt-topic contribution's \`topics\``;
}
} catch (err) {
return {
what,
result: "refused",
problem: `the broker at ${want.host}:${want.port} could not be asked: ${scrub(err, want.password)}; nothing was written`,
};
}
try {
const entries = (await deps.hass.entries("mqtt")).filter((e) => e.domain === "mqtt");
if (entries.length > 1) {
return { what, result: "refused", problem: `Home Assistant has ${entries.length} MQTT entries; which one the mesh owns is not guessed` };
}
if (entries.length === 0) return await createEntry(deps, want, note);
const entry = entries[0];
const flow = await deps.hass.startFlow("mqtt", entry.entry_id);
if (flow.type !== "form" || !flow.flow_id || !flow.data_schema) {
if (flow.flow_id) await deps.hass.abortFlow(flow.flow_id);
return { what, result: "refused", problem: `Home Assistant's MQTT reconfigure flow answered ${flow.type}${flow.reason ? ` (${flow.reason})` : ""}` };
}
const current = formValues(flow.data_schema);
const fields: string[] = [];
if (String(current.broker ?? "") !== want.host) fields.push("broker");
if (Number(current.port ?? 0) !== want.port) fields.push("port");
if (String(current.username ?? "") !== want.username) fields.push("username");
if ((await deps.marks.get("mqtt")) !== markFor(entry.entry_id, want)) fields.push("password");
if (fields.length === 0) {
await deps.hass.abortFlow(flow.flow_id);
return note ? { what, result: "unchanged", note } : { what, result: "unchanged" };
}
const saved = await deps.hass.stepFlow(flow.flow_id, {
...current,
broker: want.host,
port: want.port,
username: want.username,
password: want.password,
});
if (saved.type === "abort" && saved.reason === "reconfigure_successful") {
await deps.marks.set("mqtt", markFor(entry.entry_id, want));
return { what, result: "written", fields, ...(note ? { note } : {}) };
}
if (saved.flow_id) await deps.hass.abortFlow(saved.flow_id);
return {
what,
result: "refused",
problem:
`Home Assistant's own connection test refused ${want.username}@${want.host}:${want.port} ` +
`(${describe(saved)}); its MQTT entry is unchanged`,
};
} catch (err) {
return { what, result: "refused", problem: scrub(err, want.password) };
}
}
/** A fresh Home Assistant has no MQTT entry: made through the integration's user flow. */
async function createEntry(deps: MqttDeps, want: MqttWanted, note?: string): Promise<Outcome> {
const what = "mqtt";
let flow = await deps.hass.startFlow("mqtt");
if (flow.type === "form" && flow.step_id !== "broker" && flow.flow_id) {
// Anything before the broker form (none outside the Supervisor) is not this step's to answer.
await deps.hass.abortFlow(flow.flow_id);
return { what, result: "refused", problem: `Home Assistant's MQTT user flow asked ${flow.step_id} before the broker` };
}
if (flow.type !== "form" || !flow.flow_id) {
return { what, result: "refused", problem: `Home Assistant's MQTT user flow answered ${describe(flow)}` };
}
const shown = formValues(flow.data_schema);
// A new entry's form has no value for its two certificate choices (a reconfigure pre-fills them
// from the entry): plain MQTT, so neither a CA nor a client certificate.
const other = (shown.other_settings ?? {}) as Record<string, unknown>;
if (flow.data_schema?.some((f) => f.name === "other_settings")) {
shown.other_settings = { set_ca_cert: "off", set_client_cert: false, ...other };
}
flow = await deps.hass.stepFlow(flow.flow_id, {
...shown,
broker: want.host,
port: want.port,
username: want.username,
password: want.password,
});
if (flow.type === "create_entry") {
const id = (flow.result as { entry_id?: string } | undefined)?.entry_id;
if (id) await deps.marks.set("mqtt", markFor(id, want));
return { what, result: "written", fields: ["entry"], ...(note ? { note } : {}) };
}
if (flow.flow_id) await deps.hass.abortFlow(flow.flow_id);
return { what, result: "refused", problem: `Home Assistant refused a new MQTT entry for ${want.host}:${want.port} (${describe(flow)})` };
}
export function describe(r: { type: string; reason?: string; errors?: Record<string, string> | null }): string {
const errors = r.errors ? Object.entries(r.errors).map(([k, v]) => `${k}: ${v}`).join(", ") : "";
return [r.type, r.reason, errors].filter(Boolean).join(" — ");
}
// ---- Sonarr, Radarr, Lidarr ----
// Home Assistant's Sonarr, Radarr and Lidarr integrations, pointed at the apps the mesh bound —
// `sonarr-api`, `radarr-api`, `lidarr-api` (their providers: mesh-catalog #156).
//
// **What Home Assistant lets anyone change, and what it does not.** Each integration keeps a URL and
// an API key in its config entry. None of the three has a reconfigure flow: Home Assistant changes
// them only through the flow its UI runs —
// - a **user flow** makes a new entry (validated against the app);
// - a **reauth flow**, which Home Assistant starts by itself when the app refuses the key it holds,
// takes a new key (Sonarr) or a new URL and key (Radarr, Lidarr);
// - anything else — the URL of a working entry — only by removing the integration and adding it
// again, which throws away its entities' names, areas and history links. **This step never
// removes an entry.**
// So, per app:
// 1. The bound key is tried against the bound app first. Refused, nothing is written: until the
// operator accepts the app's own key for this pair, the mesh delivers a value it minted, which
// no Servarr app takes (novox/hq ADR 0092) — the failure names the `secret accept` that fixes it.
// 2. No entry: one is made through the user flow.
// 3. A reauth flow Home Assistant started for the entry: finished with the bound key (and URL,
// where the integration's reauth asks for one).
// 4. An entry whose URL (read from the device the integration registered, `configuration_url`)
// is the bound one and which is loaded: already as the mesh says. The key needs no digest here:
// a Servarr app has one key, so an entry loaded against the app holds the key the app took.
// 5. A working entry at a different URL that reaches **the same app** — the same process, by the
// app's own status (start time, data folder, version) — is left as it is and said: ace's entries
// say `127.0.0.1:<port>` and the binding says `ace.internal:<port>`, one Sonarr either way.
// 6. Anything else is refused, loudly, with what the operator can do; nothing is removed.
export interface ServarrApp {
/** The integration's domain, also the app. */
domain: "sonarr" | "radarr" | "lidarr";
/** The provision it is required as: the `requires`, `binds` and `secrets` key. */
provision: string;
/** The app's status endpoint: answers 401 to a wrong key, and says which process answered. */
statusPath: string;
}
export const APPS: readonly ServarrApp[] = [
{ domain: "sonarr", provision: "sonarr-api", statusPath: "/api/v3/system/status" },
{ domain: "radarr", provision: "radarr-api", statusPath: "/api/v3/system/status" },
{ domain: "lidarr", provision: "lidarr-api", statusPath: "/api/v1/system/status" },
];
/** The HTTP the step needs toward the apps, so a test can stand fakes in. */
export interface Http {
fetch(url: string, init?: { method?: string; headers?: Record<string, string> }): Promise<{ status: number; text(): Promise<string> }>;
}
export type AppWanted = { ok: true; url: string; key: string; from: string } | { ok: false; problem: string };
/** The URL and key the mesh says Home Assistant uses for this app. */
export function wantedApp(spec: ServarrApp, binding: Binding | undefined, credential: string | undefined): AppWanted {
if (!binding) return { ok: false, problem: `no binding for ${spec.provision} was delivered — the mesh writes it before this step runs` };
const at = typeof binding.at === "string" ? binding.at.trim() : "";
if (!at) return { ok: false, problem: `the ${spec.provision} binding names no host (at)` };
const port = portOf(binding.serves);
if (port === undefined) return { ok: false, problem: `the ${spec.provision} binding serves no usable port (${String(binding.serves?.port)})` };
const scheme = typeof binding.serves?.scheme === "string" && binding.serves.scheme ? binding.serves.scheme : "http";
if (scheme !== "http" && scheme !== "https") return { ok: false, problem: `the ${spec.provision} binding serves scheme ${scheme}` };
const base = typeof binding.serves?.["url-base"] === "string" ? String(binding.serves["url-base"]).trim().replace(/^\/+|\/+$/g, "") : "";
const key = (credential ?? "").trim();
if (!key) return { ok: false, problem: `the ${spec.provision} credential is empty or was not delivered` };
return {
ok: true,
url: `${scheme}://${urlHost(at)}:${port}${base ? `/${base}` : ""}`,
key,
from: typeof binding.from === "string" ? binding.from : "",
};
}
/** Two URLs naming the same place: scheme, host, port (explicit or default) and base path. */
export function sameUrl(a: string | null | undefined, b: string): boolean {
if (!a) return false;
try {
const x = new URL(a);
const y = new URL(b);
const port = (u: URL) => u.port || (u.protocol === "https:" ? "443" : "80");
const path = (u: URL) => u.pathname.replace(/\/+$/, "");
return x.protocol === y.protocol && x.hostname.toLowerCase() === y.hostname.toLowerCase() && port(x) === port(y) && path(x) === path(y);
} catch {
return false;
}
}
type Status = { taken: true; status: Record<string, unknown> } | { taken: false };
/** The app's status with this key: `taken: false` when it refuses the key; throws when it cannot be asked. */
export async function appStatus(http: Http, spec: ServarrApp, url: string, key: string): Promise<Status> {
const res = await http.fetch(`${url.replace(/\/+$/, "")}${spec.statusPath}`, {
method: "GET",
headers: { "X-Api-Key": key, Accept: "application/json" },
});
if (res.status === 401 || res.status === 403) return { taken: false };
if (res.status < 200 || res.status >= 300) throw new Error(`${spec.domain} answered ${res.status} at ${spec.statusPath}`);
return { taken: true, status: JSON.parse(await res.text()) as Record<string, unknown> };
}
/** Whether two status answers came from one running app. */
export function sameInstance(a: Record<string, unknown>, b: Record<string, unknown>): boolean {
const facts = ["startTime", "appData", "version"];
return facts.every((k) => a[k] !== undefined && a[k] !== null && a[k] === b[k]);
}
/** The remedy for a refused key, in the controller's words (ADR 0092). */
export function acceptRemedy(spec: ServarrApp, from: string): string {
return (
`${spec.domain} refuses the ${spec.provision} credential the mesh delivered, so nothing was written into ` +
`Home Assistant. A Servarr app has one API key and the mesh cannot make it: accept ${spec.domain}'s own key ` +
`for this pair — \`secret accept <this node> home-assistant ${spec.provision} --provider ${from || "<its node>"} ` +
`--from <file holding ${spec.domain}'s ApiKey>\``
);
}
export interface ServarrDeps {
hass: Hass;
http: Http;
}
/** The input a Servarr form takes: what it shows, with the URL (where asked) and the key laid over. */
function servarrInput(schema: readonly SchemaField[] | null | undefined, url: string, key: string): Record<string, unknown> {
const input = formValues(schema);
if (hasField(schema, "url")) input.url = url;
if (hasField(schema, "api_key")) input.api_key = key;
return input;
}
/** Bring Home Assistant's entry for one app in line with the mesh. Never throws. */
export async function reconcileApp(deps: ServarrDeps, spec: ServarrApp, binding: Binding | undefined, credential: string | undefined): Promise<Outcome> {
const what = spec.domain;
const w = wantedApp(spec, binding, credential);
if ("problem" in w) return { what, result: "refused", problem: w.problem };
let bound: Status;
try {
bound = await appStatus(deps.http, spec, w.url, w.key);
} catch (err) {
return { what, result: "refused", problem: `${spec.domain} could not be asked at ${w.url}: ${scrub(err, w.key)}` };
}
if (!bound.taken) return { what, result: "refused", problem: acceptRemedy(spec, w.from) };
try {
const entries = (await deps.hass.entries(spec.domain)).filter((e) => e.domain === spec.domain);
if (entries.length > 1) {
return { what, result: "refused", problem: `Home Assistant has ${entries.length} ${spec.domain} entries; which one the mesh owns is not guessed` };
}
// No entry: made, through the integration's own user flow, which validates the key itself.
if (entries.length === 0) {
const flow = await deps.hass.startFlow(spec.domain);
if (flow.type !== "form" || !flow.flow_id) return { what, result: "refused", problem: `Home Assistant's ${spec.domain} user flow answered ${describe(flow)}` };
const made = await deps.hass.stepFlow(flow.flow_id, servarrInput(flow.data_schema, w.url, w.key));
if (made.type === "create_entry") return { what, result: "written", fields: ["entry"] };
if (made.flow_id) await deps.hass.abortFlow(made.flow_id);
return { what, result: "refused", problem: `Home Assistant refused a new ${spec.domain} entry at ${w.url} (${describe(made)})` };
}
const entry = entries[0];
if (entry.disabled_by) return { what, result: "unchanged", note: `the ${spec.domain} entry is disabled (by ${entry.disabled_by}); left alone` };
// A reauth Home Assistant started because the app refused its key: finished with the bound one.
const reauth = (await deps.hass.flowsInProgress()).find(
(f) => f.handler === spec.domain && f.context?.source === "reauth" && f.context?.entry_id === entry.entry_id,
);
if (reauth) {
let step = await deps.hass.stepFlow(reauth.flow_id, {}); // reauth_confirm: a confirmation, no fields
if (step.type === "form" && step.flow_id && step.step_id !== "reauth_confirm") {
const input = servarrInput(step.data_schema, w.url, w.key);
const fields = ["api_key", ...(hasField(step.data_schema, "url") ? ["url"] : [])];
step = await deps.hass.stepFlow(step.flow_id, input);
if (step.type === "abort" && step.reason === "reauth_successful") return { what, result: "written", fields };
}
return { what, result: "refused", problem: `Home Assistant's ${spec.domain} reauth did not take the bound key and URL (${describe(step)})` };
}
const device = (await deps.hass.devices()).find((d) => d.config_entries?.includes(entry.entry_id) && d.configuration_url);
const current = device?.configuration_url ?? undefined;
if (entry.state === "loaded" && sameUrl(current, w.url)) return { what, result: "unchanged" };
if (entry.state === "loaded" && current) {
let there: Status | undefined;
try {
there = await appStatus(deps.http, spec, current, w.key);
} catch {
there = undefined;
}
if (there?.taken && sameInstance(there.status, bound.status)) {
return {
what,
result: "equivalent",
note:
`Home Assistant reaches ${spec.domain} at ${current}, the same running app the mesh bound at ${w.url}; ` +
`Home Assistant has no way to change a working ${spec.domain} entry's URL short of removing it, so it is left as it is`,
};
}
}
return {
what,
result: "refused",
problem:
`Home Assistant's ${spec.domain} entry (${entry.state ?? "unknown state"}) points at ${current ?? "an unknown URL"}, ` +
`not the ${spec.domain} the mesh bound at ${w.url}. Home Assistant only lets a working entry's URL change by ` +
`removing and re-adding the integration, which this step never does: remove it in Home Assistant ` +
`(Settings → Devices & services → ${spec.domain}) and the next run adds it at the bound URL`,
};
} catch (err) {
return { what, result: "refused", problem: scrub(err, w.key) };
}
}
-160
View File
@@ -1,160 +0,0 @@
// Home Assistant's own configuration API, as the provisions step uses it — the supported way to
// change an integration's connection. Home Assistant keeps every integration in
// `.storage/core.config_entries`, a file it owns and rewrites; the mesh may not write it, and it is
// not a file the mesh could merge into. What Home Assistant offers instead is the same thing its UI
// uses: **config flows** over REST (`/api/config/config_entries/flow`) — a user flow creates an
// entry, a reconfigure flow changes one, a reauth flow (which Home Assistant starts itself when a
// credential stops working) replaces its credential — each validated by the integration's own
// connection test before anything is saved. The two things REST does not answer (which flows Home
// Assistant has started, which device an entry made) come over its WebSocket API.
//
// Nothing here reads `.storage`. Authenticated with the module's accepted long-lived access token.
/** A config entry as `GET /api/config/config_entries/entry` lists it — no data, no credentials. */
export interface ConfigEntry {
entry_id: string;
domain: string;
title?: string;
source?: string;
state?: string;
disabled_by?: string | null;
}
/** One field of a flow's form, as Home Assistant serializes a voluptuous schema. */
export interface SchemaField {
name: string;
type?: string;
required?: boolean;
optional?: boolean;
default?: unknown;
description?: { suggested_value?: unknown } | null;
/** A section (`type: "expandable"`) carries its own fields. */
schema?: SchemaField[];
}
/** What a flow answered: another form, an entry made, or the flow ended (abort). */
export interface FlowResult {
type: string;
flow_id?: string;
handler?: string;
step_id?: string;
data_schema?: SchemaField[] | null;
errors?: Record<string, string> | null;
reason?: string;
result?: { entry_id?: string } | unknown;
}
/** A flow in progress that Home Assistant started itself (a reauth, a discovery). */
export interface FlowProgress {
flow_id: string;
handler: string;
step_id?: string;
context?: { source?: string; entry_id?: string };
}
/** A device from the device registry; an integration names where its app is as configuration_url. */
export interface DeviceEntry {
id: string;
config_entries?: string[];
configuration_url?: string | null;
}
export interface Hass {
entries(domain: string): Promise<ConfigEntry[]>;
/** A user flow for `handler`, or — given an entry — a reconfigure flow for it. */
startFlow(handler: string, entryId?: string): Promise<FlowResult>;
stepFlow(flowId: string, input: Record<string, unknown>): Promise<FlowResult>;
abortFlow(flowId: string): Promise<void>;
flowsInProgress(): Promise<FlowProgress[]>;
devices(): Promise<DeviceEntry[]>;
}
/** Home Assistant over HTTP: REST for entries and flows, one short WebSocket session per question. */
export class HassApi implements Hass {
private readonly base: string;
constructor(url: string, private readonly token: string) {
this.base = url.replace(/\/$/, "");
}
private async rest(method: string, path: string, body?: unknown): Promise<unknown> {
const res = await fetch(`${this.base}${path}`, {
method,
headers: {
Authorization: `Bearer ${this.token}`,
Accept: "application/json",
...(body !== undefined ? { "Content-Type": "application/json" } : {}),
},
body: body !== undefined ? JSON.stringify(body) : undefined,
});
const text = await res.text();
if (!res.ok) {
// Home Assistant's error text names fields, never echoes their values.
throw new Error(`Home Assistant ${method} ${path} answered ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`);
}
return text ? (JSON.parse(text) as unknown) : undefined;
}
async entries(domain: string): Promise<ConfigEntry[]> {
return ((await this.rest("GET", `/api/config/config_entries/entry?domain=${encodeURIComponent(domain)}`)) ??
[]) as ConfigEntry[];
}
async startFlow(handler: string, entryId?: string): Promise<FlowResult> {
return (await this.rest("POST", "/api/config/config_entries/flow", {
handler,
show_advanced_options: true,
...(entryId ? { entry_id: entryId } : {}),
})) as FlowResult;
}
async stepFlow(flowId: string, input: Record<string, unknown>): Promise<FlowResult> {
return (await this.rest("POST", `/api/config/config_entries/flow/${encodeURIComponent(flowId)}`, input)) as FlowResult;
}
async abortFlow(flowId: string): Promise<void> {
await this.rest("DELETE", `/api/config/config_entries/flow/${encodeURIComponent(flowId)}`).catch(() => undefined);
}
async flowsInProgress(): Promise<FlowProgress[]> {
return (await this.ws("config_entries/flow/progress")) as FlowProgress[];
}
async devices(): Promise<DeviceEntry[]> {
return (await this.ws("config/device_registry/list")) as DeviceEntry[];
}
/** One WebSocket command: connect, authenticate, ask, close. */
private ws(type: string): Promise<unknown> {
const url = `${this.base.replace(/^http/, "ws")}/api/websocket`;
return new Promise((resolve, reject) => {
const socket = new WebSocket(url);
const timer = setTimeout(() => {
socket.close();
reject(new Error(`Home Assistant's WebSocket did not answer ${type} within 30s`));
}, 30_000);
const done = (fn: () => void): void => {
clearTimeout(timer);
socket.close();
fn();
};
socket.onerror = () => done(() => reject(new Error(`Home Assistant's WebSocket at ${url} failed`)));
socket.onmessage = (event: { data: unknown }) => {
const msg = JSON.parse(String(event.data)) as {
type: string;
id?: number;
success?: boolean;
result?: unknown;
error?: { message?: string };
};
if (msg.type === "auth_required") socket.send(JSON.stringify({ type: "auth", access_token: this.token }));
else if (msg.type === "auth_invalid") done(() => reject(new Error("Home Assistant refused the token")));
else if (msg.type === "auth_ok") socket.send(JSON.stringify({ id: 1, type }));
else if (msg.type === "result" && msg.id === 1) {
if (msg.success) done(() => resolve(msg.result));
else done(() => reject(new Error(`Home Assistant ${type}: ${msg.error?.message ?? "failed"}`)));
}
};
});
}
}
@@ -1,84 +0,0 @@
// home-assistant's provisions step — run once by the host after Home Assistant starts, and again
// whenever a binding or pair credential it reads changes (the container's `restart-on`, novox/hq
// ADR 0099). It points Home Assistant's MQTT integration at the `mqtt-topic` broker and its Sonarr,
// Radarr and Lidarr integrations at the `sonarr-api`, `radarr-api` and `lidarr-api` apps, through
// Home Assistant's own config flows (connections.ts). It connects to no mesh broker.
//
// Exits non-zero when anything could not be put right, so the node reports the step failed and the
// host runs it again on the next apply. Declared last in the manifest, so its failing gates nothing
// else of home-assistant's (novox/hq ADR 0136). Never prints a key or password.
import { join } from "node:path";
import { APPS, MQTT_PROVISION, reconcileApp, reconcileMqtt, type Outcome } from "./connections.js";
import { HassApi } from "./hass.js";
import { marksIn, readBinding, readIfThere } from "./mesh.js";
import { probeBroker } from "./probe.js";
const dir = process.env.MESH_PROVISIONS_DIR ?? "/run/provisions";
const url = process.env.MESH_HOMEASSISTANT_URL ?? "http://127.0.0.1:8123";
const token = (await readIfThere(process.env.MESH_HOMEASSISTANT_TOKEN_FILE))?.trim() ?? "";
const marks = marksIn(process.env.MESH_WRITTEN_DIR ?? "/var/lib/home-assistant-provisions");
const waitSeconds = Number(process.env.MESH_HOMEASSISTANT_WAIT_SECONDS ?? "300");
if (!token) {
console.error("[hass-provisions] no Home Assistant token — home-assistant's own `token` secret has not been accepted");
process.exit(1);
}
/** Home Assistant answers /api/ with 200 once it is up and the token is good. */
async function ready(): Promise<boolean> {
const until = Date.now() + waitSeconds * 1000;
for (;;) {
try {
const res = await fetch(`${url.replace(/\/$/, "")}/api/`, { headers: { Authorization: `Bearer ${token}` } });
if (res.status === 200) return true;
if (res.status === 401 || res.status === 403) {
console.error("[hass-provisions] Home Assistant refuses the token — accept a long-lived access token it issued");
return false;
}
} catch {
// not listening yet
}
if (Date.now() >= until) return false;
await new Promise((r) => setTimeout(r, 3000));
}
}
if (!(await ready())) {
console.error(`[hass-provisions] Home Assistant did not answer at ${url} within ${waitSeconds}s`);
process.exit(1);
}
const hass = new HassApi(url, token);
const read = async (p: string) => [await readBinding(join(dir, `${p}.json`)), await readIfThere(join(dir, `${p}.secret`))] as const;
const outcomes: Outcome[] = [];
{
const [binding, secret] = await read(MQTT_PROVISION);
outcomes.push(await reconcileMqtt({ hass, probe: probeBroker, marks }, binding, secret));
}
for (const spec of APPS) {
const [binding, secret] = await read(spec.provision);
outcomes.push(await reconcileApp({ hass, http: { fetch: (u, init) => fetch(u, init) } }, spec, binding, secret));
}
let failed = 0;
for (const o of outcomes) {
switch (o.result) {
case "unchanged":
console.log(`[hass-provisions] ${o.what}: already as the mesh says${o.note ? ` — ${o.note}` : ""}`);
break;
case "written":
console.log(`[hass-provisions] ${o.what}: wrote ${o.fields.join(", ")}; Home Assistant's own test passed${o.note ? ` — ${o.note}` : ""}`);
break;
case "equivalent":
console.log(`[hass-provisions] ${o.what}: ${o.note}`);
break;
case "refused":
failed++;
console.error(`[hass-provisions] ${o.what}: ${o.problem}`);
break;
}
}
process.exitCode = failed > 0 ? 1 : 0;
-42
View File
@@ -1,42 +0,0 @@
// What the mesh delivered to home-assistant's provisions step, and the step's own small memory.
//
// Per provision it requires, the mesh writes two files beside each other (the manifest's `binds` and
// `secrets`): `<provision>.json`, the binding — where the provider is (`at`), what it serves (`port`,
// `scheme`, …) and the login this module presents (`as`) — and `<provision>.secret`, the pair
// credential. Nothing here guesses a host, a port or a key.
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { Binding, Marks } from "./connections.js";
/** A file the mesh wrote, or undefined when it is not there. */
export async function readIfThere(path: string | undefined): Promise<string | undefined> {
if (!path) return undefined;
return readFile(path, "utf8").catch(() => undefined);
}
/** A binding file parsed, or undefined when absent or not JSON. */
export async function readBinding(path: string): Promise<Binding | undefined> {
const raw = await readIfThere(path);
if (raw === undefined) return undefined;
try {
return JSON.parse(raw) as Binding;
} catch {
return undefined;
}
}
export function marksIn(dir: string): Marks {
return {
async get(name) {
return (await readIfThere(join(dir, `${name}.digest`)))?.trim() || undefined;
},
async set(name, value) {
await mkdir(dir, { recursive: true, mode: 0o700 });
const path = join(dir, `${name}.digest`);
await writeFile(`${path}.tmp`, `${value}\n`, { mode: 0o600 });
await rename(`${path}.tmp`, path);
},
};
}
-117
View File
@@ -1,117 +0,0 @@
// Ask the broker, before Home Assistant is told anything, whether it takes the login and password
// the mesh delivered — and whether that login may subscribe to Home Assistant's discovery topics.
//
// One MQTT 3.1.1 session: CONNECT (clean, a throwaway client id, so Home Assistant's own session is
// never taken over), read the CONNACK, optionally SUBSCRIBE once and read the SUBACK, DISCONNECT.
// No dependency: the handful of bytes MQTT needs for this are written here.
import { randomBytes } from "node:crypto";
import { connect } from "node:net";
export interface ProbeResult {
/** 0 accepted; 4 bad username or password; 5 not authorised. */
connack: number;
/** The SUBACK return code for the filter asked about: 0–2 granted, 0x80 refused. */
suback?: number;
}
export type Probe = (host: string, port: number, username: string, password: string, subscribe?: string) => Promise<ProbeResult>;
function str(v: string): Buffer {
const b = Buffer.from(v, "utf8");
const len = Buffer.alloc(2);
len.writeUInt16BE(b.length);
return Buffer.concat([len, b]);
}
function packet(type: number, body: Buffer): Buffer {
let remaining = body.length;
const lenBytes: number[] = [];
do {
let byte = remaining % 128;
remaining = Math.floor(remaining / 128);
if (remaining > 0) byte |= 0x80;
lenBytes.push(byte);
} while (remaining > 0);
return Buffer.concat([Buffer.from([type, ...lenBytes]), body]);
}
/** The first complete packet in `buf`: its type byte, its body, and how many bytes it took. */
export function firstPacket(buf: Buffer): { type: number; body: Buffer; used: number } | undefined {
if (buf.length < 2) return undefined;
let length = 0;
let multiplier = 1;
let i = 1;
for (;;) {
if (i >= buf.length) return undefined;
const byte = buf[i++];
length += (byte & 0x7f) * multiplier;
if ((byte & 0x80) === 0) break;
multiplier *= 128;
if (i > 4) throw new Error("malformed MQTT remaining length");
}
if (buf.length < i + length) return undefined;
return { type: buf[0], body: buf.subarray(i, i + length), used: i + length };
}
export const probeBroker: Probe = (host, port, username, password, subscribe) => {
const connectBody = Buffer.concat([
str("MQTT"),
Buffer.from([4, 0xc2, 0, 10]), // level 4 (3.1.1); username + password + clean session; keepalive 10s
str(`mesh-probe-${randomBytes(6).toString("hex")}`),
str(username),
str(password),
]);
return new Promise((resolve, reject) => {
const socket = connect({ host, port });
let buf = Buffer.alloc(0);
const result: ProbeResult = { connack: -1 };
const timer = setTimeout(() => {
socket.destroy();
reject(new Error(`no answer from the broker at ${host}:${port} within 10s`));
}, 10_000);
const finish = (): void => {
clearTimeout(timer);
if (result.connack === 0) socket.end(Buffer.from([0xe0, 0]));
else socket.destroy();
resolve(result);
};
socket.on("connect", () => socket.write(packet(0x10, connectBody)));
socket.on("data", (chunk) => {
buf = Buffer.concat([buf, chunk]);
for (;;) {
let p;
try {
p = firstPacket(buf);
} catch (err) {
clearTimeout(timer);
socket.destroy();
reject(err);
return;
}
if (!p) return;
buf = buf.subarray(p.used);
const kind = p.type >> 4;
if (kind === 2) {
result.connack = p.body[1] ?? -1;
if (result.connack !== 0 || !subscribe) return finish();
// SUBSCRIBE, packet id 1, one filter at QoS 0.
socket.write(packet(0x82, Buffer.concat([Buffer.from([0, 1]), str(subscribe), Buffer.from([0])])));
} else if (kind === 9) {
result.suback = p.body[2];
return finish();
}
}
});
socket.on("error", (err) => {
clearTimeout(timer);
reject(err);
});
socket.on("close", () => {
if (result.connack === -1) {
clearTimeout(timer);
reject(new Error(`the broker at ${host}:${port} closed the connection without answering`));
}
});
});
};
@@ -1,293 +0,0 @@
// What holds home-assistant's provisions step (provisions/*.ts): Home Assistant's MQTT entry is
// made to use the broker, port and login the mesh bound — only after the broker takes that login,
// through the reconfigure flow, keeping every other setting as Home Assistant pre-filled it, and not
// again once it already says so; its Sonarr/Radarr/Lidarr entries are made, finished (reauth), left
// alone when they already reach the bound app, and never removed; a key the app refuses (the mesh's
// minted value before the operator accepts the app's) is never written.
//
// Home Assistant and the apps are fakes answering as the real ones do (flow shapes checked against
// ghcr.io/home-assistant/home-assistant 2026.9.3, the build ace runs).
import { test } from "node:test";
import assert from "node:assert/strict";
import type { ConfigEntry, DeviceEntry, FlowProgress, FlowResult, Hass, SchemaField } from "../provisions/hass.ts";
import type { Binding, Marks } from "../provisions/connections.ts";
import { APPS, formValues, reconcileApp, reconcileMqtt, sameUrl, type Http, type ServarrApp } from "../provisions/connections.ts";
import type { Probe } from "../provisions/probe.ts";
const PWD_NOT_CHANGED = "__**password_not_changed**__";
const MINTED = "mesh-minted-password";
function mqttBinding(): Binding {
return { provision: "mqtt-topic", from: "ace", at: "ace.internal", as: "mesh_ace_hass", serves: { scheme: "mqtt", port: 1883 } };
}
/** The MQTT reconfigure form as Home Assistant serializes it, pre-filled from an entry. */
function brokerForm(data: Record<string, unknown>): SchemaField[] {
return [
{ name: "broker", type: "string", required: true, description: { suggested_value: data.broker } },
{ name: "port", type: "integer", required: true, default: 1883, description: { suggested_value: data.port } },
{ name: "protocol", type: "select", required: true, default: "3.1.1", description: { suggested_value: data.protocol } },
{ name: "username", type: "string", optional: true, description: { suggested_value: data.username } },
{ name: "password", type: "string", optional: true, description: { suggested_value: data.password ? PWD_NOT_CHANGED : undefined } },
{
name: "other_settings",
type: "expandable",
required: true,
schema: [
{ name: "keepalive", type: "integer", optional: true, description: { suggested_value: 60 } },
{ name: "transport", type: "select", required: true, default: "tcp", description: { suggested_value: "tcp" } },
{ name: "set_ca_cert", type: "select", required: true, description: { suggested_value: "off" } },
{ name: "set_client_cert", type: "boolean", required: true, description: { suggested_value: false } },
],
},
];
}
interface FakeOpts {
entries?: Record<string, (ConfigEntry & { data: Record<string, unknown> })[]>;
/** What Home Assistant's own connection test accepts. */
accepts?: (data: Record<string, unknown>) => boolean;
reauth?: FlowProgress[];
devices?: DeviceEntry[];
}
function fakeHass(opts: FakeOpts = {}) {
const entries = opts.entries ?? {};
const calls: string[] = [];
const submitted: Record<string, unknown>[] = [];
const flows = new Map<string, { handler: string; entryId?: string; step: string; reauth?: boolean }>();
let n = 0;
const accepts = opts.accepts ?? (() => true);
const form = (id: string, step: string, schema: SchemaField[], errors?: Record<string, string>): FlowResult => ({
type: "form", flow_id: id, step_id: step, data_schema: schema, errors: errors ?? null,
});
const servarrUser: SchemaField[] = [
{ name: "url", type: "string", required: true },
{ name: "api_key", type: "string", required: true },
{ name: "more_options", type: "expandable", required: true, schema: [{ name: "verify_ssl", type: "boolean", optional: true, default: false }] },
];
const hass: Hass = {
async entries(domain) {
calls.push(`entries ${domain}`);
return (entries[domain] ?? []).map(({ data: _d, ...e }) => e);
},
async startFlow(handler, entryId) {
calls.push(`start ${handler}${entryId ? ` ${entryId}` : ""}`);
const id = `f${++n}`;
if (handler === "mqtt") {
const entry = entryId ? entries.mqtt.find((e) => e.entry_id === entryId) : undefined;
if (entryId && !entry) return { type: "abort", reason: "not_found" };
flows.set(id, { handler, entryId, step: "broker" });
return form(id, "broker", brokerForm(entry?.data ?? {}));
}
if (entryId) return { type: "abort", reason: "not_implemented" }; // no reconfigure for Servarr
flows.set(id, { handler, step: "user" });
return form(id, "user", servarrUser);
},
async stepFlow(flowId, input) {
calls.push(`step ${flowId}`);
const flow = flows.get(flowId);
if (!flow) throw new Error(`Home Assistant POST flow/${flowId} answered 404`);
if (flow.step === "reauth_confirm") {
flow.step = "user";
return form(flowId, "user", [
{ name: "url", type: "string", required: true, default: "http://old:1" },
{ name: "api_key", type: "string", optional: true },
{ name: "verify_ssl", type: "boolean", optional: true, default: false },
]);
}
submitted.push(input);
if (flow.handler === "mqtt") {
const entry = entries.mqtt?.find((e) => e.entry_id === flow.entryId);
const data = { ...input, ...(input.password === PWD_NOT_CHANGED ? { password: entry?.data.password } : {}) };
if (!accepts(data)) return form(flowId, "broker", brokerForm(data), { base: "cannot_connect" });
flows.delete(flowId);
if (entry) {
entry.data = data;
return { type: "abort", reason: "reconfigure_successful" };
}
(entries.mqtt ??= []).push({ entry_id: "new-mqtt", domain: "mqtt", state: "loaded", data });
return { type: "create_entry", result: { entry_id: "new-mqtt" } };
}
if (!accepts(input)) return form(flowId, "user", servarrUser, { base: "invalid_auth" });
flows.delete(flowId);
if (flow.reauth) return { type: "abort", reason: "reauth_successful" };
(entries[flow.handler] ??= []).push({ entry_id: `new-${flow.handler}`, domain: flow.handler, state: "loaded", data: input });
return { type: "create_entry", result: { entry_id: `new-${flow.handler}` } };
},
async abortFlow(flowId) {
calls.push(`abort ${flowId}`);
flows.delete(flowId);
},
async flowsInProgress() {
for (const f of opts.reauth ?? []) flows.set(f.flow_id, { handler: f.handler, entryId: f.context?.entry_id, step: "reauth_confirm", reauth: true });
return opts.reauth ?? [];
},
async devices() {
return opts.devices ?? [];
},
};
return { hass, calls, submitted, entries };
}
function memoryMarks(): Marks & { store: Map<string, string> } {
const store = new Map<string, string>();
return { store, get: async (k) => store.get(k), set: async (k, v) => void store.set(k, v) };
}
const takes = (suback = 0): Probe => async (_h, _p, user, pass) => ({ connack: user === "mesh_ace_hass" && pass === MINTED ? 0 : 5, suback });
const aceMqttEntry = () => ({
entry_id: "7d1e", domain: "mqtt", state: "loaded",
data: { broker: "127.0.0.1", port: 1883, protocol: "5", username: "luffy", password: "luffys-password" },
});
test("mqtt: the broker is asked first; a login it does not take is never written", async () => {
const f = fakeHass({ entries: { mqtt: [aceMqttEntry()] } });
const out = await reconcileMqtt({ hass: f.hass, probe: async () => ({ connack: 5 }), marks: memoryMarks() }, mqttBinding(), MINTED);
assert.equal(out.result, "refused");
assert.match((out as { problem: string }).problem, /does not \(yet\) take the login mesh_ace_hass/);
assert.deepEqual(f.calls, []); // Home Assistant not even asked
assert.equal(f.entries.mqtt[0].data.username, "luffy");
});
test("mqtt: ace's entry (127.0.0.1, luffy) is moved to the bound broker and login, every other setting kept", async () => {
const f = fakeHass({ entries: { mqtt: [aceMqttEntry()] } });
const marks = memoryMarks();
const out = await reconcileMqtt({ hass: f.hass, probe: takes(), marks }, mqttBinding(), `${MINTED}\n`);
assert.deepEqual(out, { what: "mqtt", result: "written", fields: ["broker", "username", "password"] });
assert.deepEqual(f.entries.mqtt[0].data, {
broker: "ace.internal", port: 1883, protocol: "5", username: "mesh_ace_hass", password: MINTED,
other_settings: { keepalive: 60, transport: "tcp", set_ca_cert: "off", set_client_cert: false },
});
assert.ok(marks.store.get("mqtt"));
assert.ok(![...marks.store.values()].some((v) => v.includes(MINTED)));
// Run again: nothing differs, the flow is opened to read and closed without submitting.
const before = f.submitted.length;
const again = await reconcileMqtt({ hass: f.hass, probe: takes(), marks }, mqttBinding(), MINTED);
assert.deepEqual(again, { what: "mqtt", result: "unchanged" });
assert.equal(f.submitted.length, before);
assert.match(f.calls.at(-1) ?? "", /^abort /);
});
test("mqtt: a new password alone is written (the digest tells)", async () => {
const f = fakeHass({ entries: { mqtt: [aceMqttEntry()] } });
const marks = memoryMarks();
await reconcileMqtt({ hass: f.hass, probe: takes(), marks }, mqttBinding(), MINTED);
const rotated: Probe = async () => ({ connack: 0, suback: 0 });
const out = await reconcileMqtt({ hass: f.hass, probe: rotated, marks }, mqttBinding(), "rotated");
assert.deepEqual(out, { what: "mqtt", result: "written", fields: ["password"] });
assert.equal(f.entries.mqtt[0].data.password, "rotated");
});
test("mqtt: Home Assistant's own connection test refusing saves nothing and fails loudly", async () => {
const f = fakeHass({ entries: { mqtt: [aceMqttEntry()] }, accepts: () => false });
const marks = memoryMarks();
const out = await reconcileMqtt({ hass: f.hass, probe: takes(), marks }, mqttBinding(), MINTED);
assert.equal(out.result, "refused");
assert.match((out as { problem: string }).problem, /cannot_connect.*unchanged/);
assert.equal(f.entries.mqtt[0].data.username, "luffy");
assert.equal(marks.store.size, 0);
});
test("mqtt: a fresh Home Assistant gets an entry; a grant without the discovery topics is warned about", async () => {
const f = fakeHass();
const out = await reconcileMqtt({ hass: f.hass, probe: takes(0x80), marks: memoryMarks() }, mqttBinding(), MINTED);
assert.equal(out.result, "written");
assert.match((out as { note?: string }).note ?? "", /may not subscribe to homeassistant\/#/);
assert.equal(f.entries.mqtt[0].data.broker, "ace.internal");
});
test("mqtt: two entries, or a binding without a port, are refused rather than guessed", async () => {
const f = fakeHass({ entries: { mqtt: [aceMqttEntry(), { ...aceMqttEntry(), entry_id: "other" }] } });
assert.equal((await reconcileMqtt({ hass: f.hass, probe: takes(), marks: memoryMarks() }, mqttBinding(), MINTED)).result, "refused");
const noPort = { ...mqttBinding(), serves: {} };
assert.match(((await reconcileMqtt({ hass: f.hass, probe: takes(), marks: memoryMarks() }, noPort, MINTED)) as { problem: string }).problem, /no usable port/);
});
// ---- Servarr ----
const SONARR = APPS.find((a) => a.domain === "sonarr") as ServarrApp;
const RADARR = APPS.find((a) => a.domain === "radarr") as ServarrApp;
const KEY = "the-apps-own-key";
const servarrBinding = (port: number, at = "ace.internal"): Binding => ({ provision: "sonarr-api", from: "ace", at, as: "mesh_ace_hass", serves: { scheme: "http", port, "url-base": "" } });
/** One running Sonarr, answering on several addresses (127.0.0.1 and ace.internal are one host). */
function apps(instances: Record<string, { startTime: string }>): Http & { asked: string[] } {
const asked: string[] = [];
return {
asked,
async fetch(url, init) {
asked.push(url);
const u = new URL(url);
const inst = instances[`${u.hostname}:${u.port}`];
if (!inst) throw new Error("connect ECONNREFUSED");
if (init?.headers?.["X-Api-Key"] !== KEY) return { status: 401, text: async () => "" };
return { status: 200, text: async () => JSON.stringify({ version: "4.0.15", appData: "/config", startTime: inst.startTime }) };
},
};
}
const oneSonarr = () => apps({ "ace.internal:8989": { startTime: "t1" }, "127.0.0.1:8989": { startTime: "t1" } });
const sonarrEntry = (state = "loaded") => ({ entry_id: "5a1d", domain: "sonarr", state, data: { url: "http://127.0.0.1:8989", api_key: KEY } });
test("servarr: the mesh's minted key is never written; the remedy names the accept", async () => {
const f = fakeHass({ entries: { sonarr: [sonarrEntry()] } });
const out = await reconcileApp({ hass: f.hass, http: oneSonarr() }, SONARR, servarrBinding(8989), "minted-by-the-mesh");
assert.equal(out.result, "refused");
assert.match((out as { problem: string }).problem, /secret accept <this node> home-assistant sonarr-api --provider ace/);
assert.deepEqual(f.calls, []);
});
test("servarr: ace's entry at 127.0.0.1 reaches the same Sonarr the mesh bound at ace.internal — left, and said", async () => {
const f = fakeHass({ entries: { sonarr: [sonarrEntry()] }, devices: [{ id: "d", config_entries: ["5a1d"], configuration_url: "http://127.0.0.1:8989" }] });
const out = await reconcileApp({ hass: f.hass, http: oneSonarr() }, SONARR, servarrBinding(8989), KEY);
assert.equal(out.result, "equivalent");
assert.equal(f.submitted.length, 0);
});
test("servarr: an entry already at the bound URL is unchanged", async () => {
const f = fakeHass({ entries: { sonarr: [sonarrEntry()] }, devices: [{ id: "d", config_entries: ["5a1d"], configuration_url: "http://ace.internal:8989" }] });
assert.deepEqual(await reconcileApp({ hass: f.hass, http: oneSonarr() }, SONARR, servarrBinding(8989), KEY), { what: "sonarr", result: "unchanged" });
});
test("servarr: a working entry that reaches a different app is refused, and nothing is removed", async () => {
const f = fakeHass({ entries: { sonarr: [sonarrEntry()] }, devices: [{ id: "d", config_entries: ["5a1d"], configuration_url: "http://127.0.0.1:8989" }] });
const two = apps({ "ace.internal:8989": { startTime: "t1" }, "127.0.0.1:8989": { startTime: "another" } });
const out = await reconcileApp({ hass: f.hass, http: two }, SONARR, servarrBinding(8989), KEY);
assert.equal(out.result, "refused");
assert.match((out as { problem: string }).problem, /never does/);
assert.equal(f.entries.sonarr.length, 1);
});
test("servarr: no entry — one is made at the bound URL through the user flow", async () => {
const f = fakeHass({ entries: {} });
const out = await reconcileApp({ hass: f.hass, http: oneSonarr() }, SONARR, servarrBinding(8989), KEY);
assert.deepEqual(out, { what: "sonarr", result: "written", fields: ["entry"] });
assert.deepEqual(f.submitted[0], { url: "http://ace.internal:8989", api_key: KEY, more_options: { verify_ssl: false } });
});
test("servarr: a reauth Home Assistant started is finished with the bound URL and key", async () => {
const entry = { ...sonarrEntry("setup_error"), domain: "radarr", entry_id: "1955" };
const f = fakeHass({
entries: { radarr: [entry] },
reauth: [{ flow_id: "r1", handler: "radarr", step_id: "reauth_confirm", context: { source: "reauth", entry_id: "1955" } }],
});
const radarr = apps({ "ace.internal:7878": { startTime: "t" } });
const out = await reconcileApp({ hass: f.hass, http: radarr }, RADARR, { ...servarrBinding(7878), provision: "radarr-api" }, KEY);
assert.deepEqual(out, { what: "radarr", result: "written", fields: ["api_key", "url"] });
assert.deepEqual(f.submitted[0], { url: "http://ace.internal:7878", api_key: KEY, verify_ssl: false });
});
test("form values: suggested first, then default, sections nested", () => {
assert.deepEqual(formValues(brokerForm({ broker: "b", port: 1, protocol: "5", username: "u", password: "p" })), {
broker: "b", port: 1, protocol: "5", username: "u", password: PWD_NOT_CHANGED,
other_settings: { keepalive: 60, transport: "tcp", set_ca_cert: "off", set_client_cert: false },
});
assert.ok(sameUrl("http://ace.internal:8989/", "http://ace.internal:8989"));
assert.ok(sameUrl("http://ACE.internal", "http://ace.internal:80"));
assert.ok(!sameUrl("http://127.0.0.1:8989", "http://ace.internal:8989"));
});
+1 -10
View File
@@ -8,14 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": [
"client.ts",
"index.ts",
"tools/index.ts",
"provisions/hass.ts",
"provisions/probe.ts",
"provisions/connections.ts",
"provisions/mesh.ts",
"provisions/index.ts"
]
"include": ["client.ts", "index.ts", "tools/index.ts"]
}
+28 -53
View File
@@ -1,26 +1,6 @@
{
"module": "icecast",
"version": "1",
"requires": [
"route",
"secret"
],
"contributes": {
"route": {
"label": "icecast",
"endpoint": "stream"
}
},
"binds": {
"route": "${dir:state}/route.json"
},
"secrets": {
"secret": {
"source": "${dir:state}/source.secret",
"admin": "${dir:state}/admin.secret",
"relay": "${dir:state}/relay.secret"
}
},
"capabilities": [
"container-runtime"
],
@@ -29,7 +9,7 @@
"stream.stopped"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/icecast/broker"
},
"listens": [
{
@@ -37,61 +17,46 @@
"port": 8000,
"protocol": "tcp",
"from": "mesh",
"why": "streams in from sources (HTTP PUT) and out to listeners, plus the status and admin pages; a public name is its route"
"why": "streams in from sources and out to listeners"
}
],
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/icecast",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/icecast-module",
"mode": "0700"
},
{
"id": "logs",
"type": "directory",
"mode": "0700",
"owner": "100:101"
},
{
"id": "server-conf",
"id": "server-env",
"type": "file",
"path": "${dir:state}/icecast.xml",
"path": "/var/lib/icecast-module/server.env",
"mode": "0600",
"content": "<icecast>\n <!-- Written by the mesh (modules/icecast). Passwords arrive as secrets rendered into this file,\n never as environment: the image's entrypoint seds ICECAST_* variables into the file only\n when they are set, and none are. -->\n <location>Earth</location>\n <admin>icemaster@localhost</admin>\n <limits>\n <clients>100</clients>\n <sources>2</sources>\n <queue-size>524288</queue-size>\n <client-timeout>30</client-timeout>\n <header-timeout>15</header-timeout>\n <source-timeout>10</source-timeout>\n <burst-on-connect>1</burst-on-connect>\n <burst-size>65535</burst-size>\n </limits>\n <authentication>\n <source-password>${secret:source}</source-password>\n <relay-password>${secret:relay}</relay-password>\n <admin-user>admin</admin-user>\n <admin-password>${secret:admin}</admin-password>\n </authentication>\n <!-- The name icecast writes into playlists (.m3u/.xspf: http://<hostname>:<port>/<mount>) and\n would announce to YP (none configured). A machine's own name belongs to its assignment, and\n an assignment merges only into JSON; this XML cannot take it, so the neutral default stays. -->\n <hostname>localhost</hostname>\n <listen-socket>\n <port>8000</port>\n </listen-socket>\n <http-headers>\n <header name=\"Access-Control-Allow-Origin\" value=\"*\" />\n </http-headers>\n <fileserve>1</fileserve>\n <paths>\n <basedir>/usr/share/icecast</basedir>\n <logdir>/var/log/icecast</logdir>\n <webroot>/usr/share/icecast/web</webroot>\n <adminroot>/usr/share/icecast/admin</adminroot>\n <alias source=\"/\" destination=\"/status.xsl\"/>\n </paths>\n <logging>\n <accesslog>access.log</accesslog>\n <errorlog>error.log</errorlog>\n <loglevel>3</loglevel>\n <logsize>10000</logsize>\n </logging>\n <security>\n <chroot>0</chroot>\n <!-- Starts as root, reads this 0600 root-owned file, then drops to the image's icecast user\n (uid 100, group icecast 101) before serving. -->\n <changeowner>\n <user>icecast</user>\n <group>icecast</group>\n </changeowner>\n </security>\n</icecast>\n"
},
{
"id": "net",
"type": "network",
"name": "icecast"
"content": "ICECAST_SOURCE_PASSWORD=${secret:source}\nICECAST_ADMIN_PASSWORD=${secret:admin}\nICECAST_RELAY_PASSWORD=${secret:relay}\nICECAST_ADMIN_USERNAME=admin\n"
},
{
"id": "server",
"type": "container",
"name": "icecast",
"image": "infiniteproject/icecast@sha256:cd506cf3dfe31ce05fd37d7e672dbd1213e7255cc93d28ecf5a3b547af4e162c",
"network": "icecast",
"env-file": [
"/var/lib/icecast-module/server.env"
],
"ports": [
"8000"
],
"volumes": [
"${dir:state}/icecast.xml:/etc/icecast.xml:ro",
"${dir:logs}:/var/log/icecast"
],
"restart-on": [
"server-conf"
]
"secrets-in-environment": "the image seds ICECAST_*_PASSWORD into icecast.xml and has no _FILE; convertible by mounting a generated icecast.xml, not yet done"
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/icecast/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -100,14 +65,14 @@
"id": "runtime",
"type": "container",
"name": "mesh-icecast",
"network": "icecast",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro"
"/var/lib/mesh/icecast/broker:/run/secrets/broker:ro",
"/var/lib/mesh/icecast/config.json:/run/config/config.json:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_ICECAST_URL": "http://icecast:8000",
"MESH_ICECAST_URL": "http://127.0.0.1:8000",
"MESH_ICECAST_CONFIG_FILE": "/run/config/config.json"
},
"restart-on": [
@@ -136,5 +101,15 @@
"from": "Dockerfile"
}
]
},
"requires": [
"secret"
],
"secrets": {
"secret": {
"source": "/var/lib/icecast-module/source.secret",
"admin": "/var/lib/icecast-module/admin.secret",
"relay": "/var/lib/icecast-module/relay.secret"
}
}
}
+2 -2
View File
@@ -13,7 +13,7 @@ ARG RUNTIME_BASE
FROM ${BUILD_BASE} AS build
WORKDIR /app/modules/influxdb
COPY . .
RUN node /app/node_modules/typescript/bin/tsc client.ts grants.ts provisioner/index.ts tools/index.ts \
RUN node /app/node_modules/typescript/bin/tsc client.ts tools/index.ts \
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
FROM ${RUNTIME_BASE}
@@ -21,4 +21,4 @@ COPY --from=build /app/modules/influxdb/dist /app/modules/influxdb/dist
# Every serve-time entrypoint, loaded by the runtime in serve mode: tools and events serve, and a
# provider's provisioner runs its reconcile loop in the same process, with the broker connected —
# the convention novox/hq issues 060/061 settled.
ENV MESH_TOOL_MODULES=/app/modules/influxdb/dist/tools/index.js,/app/modules/influxdb/dist/provisioner/index.js
ENV MESH_TOOL_MODULES=/app/modules/influxdb/dist/tools/index.js
+3 -118
View File
@@ -17,25 +17,6 @@ export interface InfluxBucket {
retentionSeconds?: number;
}
/** One permission of an authorization, as InfluxDB represents it: an action on a resource type,
* in one org, optionally narrowed to one resource by id (no id = every resource of that type). */
export interface InfluxPermission {
action: "read" | "write";
resource: { type: string; orgID?: string; id?: string; name?: string; org?: string };
}
/** A v1-compatibility ("legacy") authorization: a username (InfluxDB calls it `token`) and a
* password the caller chooses, scoped by permissions. The one credential InfluxDB 2.x lets a
* caller set to a value it did not generate — which is what a mesh-minted password needs. */
export interface LegacyAuthorization {
id: string;
token: string;
orgID: string;
status?: "active" | "inactive";
description?: string;
permissions: InfluxPermission[];
}
/** The settings-merged config the mesh delivers (novox/hq ADR 0046): { url, apiKey, token, password, user, ... }. */
function meshConfig(file?: string): Record<string, string> {
if (!file) return {};
@@ -43,20 +24,13 @@ function meshConfig(file?: string): Record<string, string> {
catch { return {}; }
}
/** A secret delivered as a file, trimmed; undefined when there is none, so the caller can fall back. */
function tokenFromFile(file?: string): string | undefined {
if (!file) return undefined;
try { return readFileSync(file, "utf8").trim() || undefined; }
catch { return undefined; }
}
export class InfluxDBClient {
readonly baseUrl: string;
constructor(
url: string,
private readonly token: string,
readonly org: string,
private readonly org: string,
) {
this.baseUrl = url.replace(/\/$/, "");
}
@@ -69,10 +43,8 @@ export class InfluxDBClient {
static fromEnv(env: NodeJS.ProcessEnv = process.env): InfluxDBClient {
const cfg = meshConfig(env.MESH_INFLUXDB_CONFIG_FILE);
const url = cfg.url ?? env.MESH_INFLUXDB_URL ?? `http://127.0.0.1:${env.INFLUXDB_PORT ?? "8086"}`;
// The token reaches the process as a file (novox/hq ADR 0086); the environment variable stays
// only for a workstation running the tools by hand.
const token = cfg.token ?? tokenFromFile(env.MESH_INFLUXDB_TOKEN_FILE) ?? env.MESH_INFLUXDB_TOKEN;
if (!token) throw new Error("no InfluxDB token — set MESH_INFLUXDB_TOKEN_FILE");
const token = cfg.token ?? env.MESH_INFLUXDB_TOKEN;
if (!token) throw new Error("no InfluxDB token — set MESH_INFLUXDB_TOKEN");
const org = cfg.org ?? env.MESH_INFLUXDB_ORG ?? "mesh";
return new InfluxDBClient(url, token, org);
}
@@ -89,93 +61,6 @@ export class InfluxDBClient {
return res;
}
/** Like request, but the answer is returned whatever its status, for the caller to read. */
private async raw(path: string, init?: RequestInit): Promise<Response> {
return fetch(`${this.baseUrl}${path}`, {
...init,
headers: { Authorization: `Token ${this.token}`, ...(init?.headers ?? {}) },
});
}
private async send(path: string, method: string, body?: unknown): Promise<Response> {
return this.request(path, {
method,
headers: { "Content-Type": "application/json" },
body: body === undefined ? undefined : JSON.stringify(body),
});
}
/** The id of the org of this name, or undefined when there is none. */
async orgID(name: string): Promise<string | undefined> {
const res = await this.raw(`/api/v2/orgs?org=${encodeURIComponent(name)}`);
if (res.status === 404) return undefined;
if (!res.ok) throw new Error(`InfluxDB API /api/v2/orgs: ${res.status} ${await res.text()}`);
const body = (await res.json()) as { orgs?: { id: string; name: string }[] };
return body.orgs?.find((o) => o.name === name)?.id;
}
/** The bucket of exactly this name in the org, or undefined. */
async findBucket(orgID: string, name: string): Promise<InfluxBucket | undefined> {
const res = await this.raw(`/api/v2/buckets?orgID=${encodeURIComponent(orgID)}&name=${encodeURIComponent(name)}`);
if (res.status === 404) return undefined;
if (!res.ok) throw new Error(`InfluxDB API /api/v2/buckets: ${res.status} ${await res.text()}`);
const body = (await res.json()) as { buckets?: { id: string; name: string; orgID?: string }[] };
const b = body.buckets?.find((x) => x.name === name);
return b ? { id: b.id, name: b.name, orgID: b.orgID } : undefined;
}
/** Create a bucket that keeps its data for ever — retention is the operator's choice, never the mesh's. */
async createBucket(orgID: string, name: string, description: string): Promise<InfluxBucket> {
const b = (await (await this.send("/api/v2/buckets", "POST", {
orgID, name, description, retentionRules: [],
})).json()) as { id: string; name: string; orgID?: string };
return { id: b.id, name: b.name, orgID: b.orgID };
}
/** The v1 authorization whose username is exactly this, or undefined. */
async findLegacy(username: string): Promise<LegacyAuthorization | undefined> {
const path = `/private/legacy/authorizations?token=${encodeURIComponent(username)}`;
const res = await this.raw(path);
// InfluxDB answers a filter matching nothing with 404, not an empty list.
if (res.status === 404) return undefined;
if (!res.ok) throw new Error(`InfluxDB API ${path}: ${res.status} ${await res.text()}`);
const body = (await res.json()) as { authorizations?: LegacyAuthorization[] };
return body.authorizations?.find((a) => a.token === username);
}
async createLegacy(a: Omit<LegacyAuthorization, "id">): Promise<LegacyAuthorization> {
return (await (await this.send("/private/legacy/authorizations", "POST", a)).json()) as LegacyAuthorization;
}
/** Set a v1 authorization's password. InfluxDB keeps only a hash of it, so it can be set, never read. */
async setLegacyPassword(id: string, password: string): Promise<void> {
await this.send(`/private/legacy/authorizations/${encodeURIComponent(id)}/password`, "POST", { password });
}
async updateLegacy(id: string, patch: { status?: "active" | "inactive"; description?: string }): Promise<void> {
await this.send(`/private/legacy/authorizations/${encodeURIComponent(id)}`, "PATCH", patch);
}
async deleteLegacy(id: string): Promise<void> {
await this.send(`/private/legacy/authorizations/${encodeURIComponent(id)}`, "DELETE");
}
/**
* Whether this username and password sign in on the v1 API — the consumer's own view. Asked with
* a statement that reads nothing (`SHOW DATABASES` lists only what the credential may read), sent
* with Basic auth so the password is never in a URL. 401 is a wrong password or no such user;
* anything else that is not a server error means InfluxDB knew who was asking.
*/
async legacySignsIn(username: string, password: string): Promise<boolean> {
const res = await fetch(`${this.baseUrl}/query?q=${encodeURIComponent("SHOW DATABASES")}`, {
headers: { Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}` },
});
await res.arrayBuffer();
if (res.status === 401) return false;
if (res.status >= 500) throw new Error(`InfluxDB v1 /query: ${res.status}`);
return true;
}
/** Server health — the one endpoint that needs no token, but we send it anyway. */
async health(): Promise<InfluxHealth> {
return (await (await this.request("/health")).json()) as InfluxHealth;
-186
View File
@@ -1,186 +0,0 @@
// What the `influxdb-api` provision means in InfluxDB: one v1-compatibility authorization per
// consumer, in the org this module serves, under the username and password the mesh gave both ends,
// allowed exactly the access the consumer contributed. The provisioner (provisioner/index.ts) is the
// sdk harness calling these; they are here, apart from it, so they can be exercised against a fake
// InfluxDB without a broker or a contributions file.
//
// **Why a v1 authorization and not a v2 API token.** The mesh mints the consumer's password and
// hands it to both ends (novox/hq ADR 0048); the provider sets it, and never hands one back. An
// InfluxDB 2.x API token is generated by the server — `POST /api/v2/authorizations` ignores a token
// the caller sends — so a token could only ever be the operator's to accept, one per pair, by hand.
// A v1 authorization is a username and a password the caller chooses (8–72 characters; the mesh
// mints 40), stored hashed, and it reads and writes through InfluxQL (`/query`) and line protocol
// (`/write`), which every bucket answers under its own name as a database (InfluxDB maps each
// bucket to a database of the same name by itself). That is what grafana's InfluxDB data source
// speaks, and what Node-RED's influxdb nodes speak in their 1.x mode — so the mesh can make every
// consumer's credential, rotate it and withdraw it, with no person in the loop.
//
// **What a consumer contributes.** `access`: "read" (the default), "write" or "read-write".
// `buckets`: the buckets it may use, by name. A reader that names none may read every bucket of the
// org — a dashboard is pointed at data, it does not own it. A writer must name its buckets: writing
// everywhere, the org's system buckets included, is never what a consumer means. A named bucket
// that does not exist is created, keeping its data for ever; the mesh never deletes a bucket.
//
// **Only what the mesh made is touched.** An authorization this module creates is named with the
// mesh's identity prefix and its description starts with MARK. One with the same username that
// lacks the mark is somebody else's: it is refused, never adopted, never updated, never deleted.
// Every other authorization, token, user and bucket in the instance is left exactly as it was.
import type { InfluxDBClient, InfluxPermission, LegacyAuthorization } from "./client.js";
/** How a description marks an authorization as the mesh's own work. */
export const MARK = "[mesh]";
/** The prefix the mesh gives every consumer identity (novox/hq ADR 0049). */
const IDENTITY_PREFIX = "mesh_";
/** One consumer, as the harness hands it over. */
export interface ApiGrant {
readonly as: string;
readonly password: string;
readonly values: Readonly<Record<string, unknown>>;
readonly consumer?: string;
}
export type Access = "read" | "write" | "read-write";
/** What a contribution asks for, checked. Refused when it cannot be served as asked. */
export function askedFor(values: Readonly<Record<string, unknown>>): { access: Access; buckets: string[] } {
const access = values.access ?? "read";
if (access !== "read" && access !== "write" && access !== "read-write") {
throw new Error(`contributes an access of ${JSON.stringify(access)} — it is "read", "write" or "read-write"`);
}
const raw = values.buckets ?? [];
if (!Array.isArray(raw) || raw.some((b) => typeof b !== "string" || b.trim() === "")) {
throw new Error(`contributes buckets of ${JSON.stringify(raw)} — a list of bucket names`);
}
const buckets = [...new Set((raw as string[]).map((b) => b.trim()))].sort();
if (access !== "read" && buckets.length === 0) {
throw new Error(`asks to write and names no bucket (\`buckets\`) — a writer names what it writes to`);
}
if (buckets.some((b) => b.startsWith("_"))) {
throw new Error(`names a system bucket (${buckets.filter((b) => b.startsWith("_")).join(", ")}) — those are InfluxDB's own`);
}
return { access: access as Access, buckets };
}
/** The permissions a grant resolves to, given each named bucket's id. */
export function permissionsFor(orgID: string, access: Access, bucketIDs: string[]): InfluxPermission[] {
const actions: ("read" | "write")[] = access === "read-write" ? ["read", "write"] : [access];
const out: InfluxPermission[] = [];
for (const action of actions) {
if (bucketIDs.length === 0) {
out.push({ action, resource: { type: "buckets", orgID } });
continue;
}
for (const id of bucketIDs) out.push({ action, resource: { type: "buckets", orgID, id } });
}
return out;
}
/** A permission as a comparable string: what InfluxDB answers carries names and links besides. */
function key(p: InfluxPermission): string {
return `${p.action}:${p.resource.type}:${p.resource.orgID ?? ""}:${p.resource.id ?? "*"}`;
}
function samePermissions(a: readonly InfluxPermission[], b: readonly InfluxPermission[]): boolean {
const x = a.map(key).sort();
const y = b.map(key).sort();
return x.length === y.length && x.every((v, i) => v === y[i]);
}
export function marked(a: Pick<LegacyAuthorization, "token" | "description">): boolean {
return a.token.startsWith(IDENTITY_PREFIX) && (a.description ?? "").startsWith(MARK);
}
function describe(g: ApiGrant): string {
return `${MARK} made by the mesh for ${g.consumer ? `a module on ${g.consumer}` : "a consumer"} — do not edit; it is reset`;
}
export class ApiGrants {
constructor(private readonly influx: InfluxDBClient, readonly org: string) {}
private async orgID(): Promise<string> {
const id = await this.influx.orgID(this.org);
if (!id) throw new Error(`InfluxDB has no org ${JSON.stringify(this.org)} — the org this module serves must exist`);
return id;
}
/** The ids of the named buckets, creating any that are missing when `create` says so. Undefined
* when one is missing and may not be created (a read-only question). */
private async bucketIDs(orgID: string, names: string[], create: ApiGrant | undefined): Promise<string[] | undefined> {
const ids: string[] = [];
for (const name of names) {
let b = await this.influx.findBucket(orgID, name);
if (!b) {
if (!create) return undefined;
b = await this.influx.createBucket(orgID, name, `${MARK} made by the mesh for ${create.as}; the mesh never deletes it`);
}
ids.push(b.id);
}
return ids.sort();
}
/** Create the consumer's authorization, or bring the mesh's existing one back to what the grant
* says. Idempotent: a second apply of the same grant changes nothing beyond re-asserting the
* password, which InfluxDB can be told but never asked. */
async ensure(g: ApiGrant): Promise<"created" | "updated" | "unchanged"> {
if (!g.as.startsWith(IDENTITY_PREFIX)) {
throw new Error(`${g.as} is not a mesh identity — the mesh names every consumer ${IDENTITY_PREFIX}<node>_<module>`);
}
const { access, buckets } = askedFor(g.values);
const orgID = await this.orgID();
const found = await this.influx.findLegacy(g.as);
if (found && !marked(found)) {
throw new Error(
`InfluxDB already has a v1 authorization ${g.as} the mesh did not make — left alone; ` +
`delete it if the mesh should own that name`);
}
const want = permissionsFor(orgID, access, (await this.bucketIDs(orgID, buckets, g))!);
if (found && found.orgID === orgID && samePermissions(found.permissions, want)) {
// Only what differs is written. The password cannot be read back, so it is tried instead.
let changed = false;
if (found.status === "inactive") {
await this.influx.updateLegacy(found.id, { status: "active" });
changed = true;
}
if (!(await this.influx.legacySignsIn(g.as, g.password))) {
await this.influx.setLegacyPassword(found.id, g.password);
changed = true;
}
return changed ? "updated" : "unchanged";
}
// InfluxDB cannot change an authorization's permissions in place, so the mesh's own is made
// again. Only ever one the mesh made: a foreign one was refused above.
if (found) await this.influx.deleteLegacy(found.id);
const made = await this.influx.createLegacy({
token: g.as, orgID, status: "active", description: describe(g), permissions: want,
});
await this.influx.setLegacyPassword(made.id, g.password);
return found ? "updated" : "created";
}
/** Whether InfluxDB still holds this consumer's authorization exactly as the grant says: present,
* the mesh's, active, allowed what was asked and nothing more, and signing in with the mesh's
* password. Reads only — a missing bucket is "not held", never created here. */
async holds(g: ApiGrant): Promise<boolean> {
const { access, buckets } = askedFor(g.values);
const orgID = await this.influx.orgID(this.org);
if (!orgID) return false;
const found = await this.influx.findLegacy(g.as);
if (!found || !marked(found) || found.status === "inactive" || found.orgID !== orgID) return false;
const ids = await this.bucketIDs(orgID, buckets, undefined);
if (!ids || !samePermissions(found.permissions, permissionsFor(orgID, access, ids))) return false;
return this.influx.legacySignsIn(g.as, g.password);
}
/** Withdraw a consumer's authorization — only one the mesh made. Its buckets and their data stay. */
async remove(as: string): Promise<"removed" | "absent" | "not ours"> {
const found = await this.influx.findLegacy(as);
if (!found) return "absent";
if (!marked(found)) return "not ours";
await this.influx.deleteLegacy(found.id);
return "removed";
}
}
+35 -63
View File
@@ -1,19 +1,11 @@
{
"module": "influxdb",
"version": "1",
"provides": [
{
"name": "influxdb-api",
"scope": "mesh"
}
],
"capabilities": [
"container-runtime"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker",
"admin": "${dir:state}/admin.secret",
"admin-token": "${dir:state}/admin-token.secret"
"broker": "/var/lib/mesh/influxdb/broker"
},
"listens": [
{
@@ -21,82 +13,64 @@
"port": 8086,
"protocol": "tcp",
"from": "mesh",
"why": "queries, writes and the web UI, over http; consumers granted influxdb-api sign in with the mesh's credential, and a name is a route grant"
"why": "queries and writes, over http"
}
],
"serves": {
"influxdb-api": {
"scheme": "http",
"port": 8086,
"org": "mesh",
"bucket": "default"
}
},
"receives": {
"influxdb-api": "${dir:grants}/mesh.json"
},
"grants": {
"influxdb-api": "${dir:grants}"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/influxdb",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/influxdb-module",
"mode": "0700"
},
{
"id": "server-env",
"type": "file",
"path": "/var/lib/influxdb-module/server.env",
"mode": "0600",
"content": "DOCKER_INFLUXDB_INIT_MODE=setup\nDOCKER_INFLUXDB_INIT_USERNAME=admin\nDOCKER_INFLUXDB_INIT_PASSWORD=${secret:admin}\nDOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${secret:admin-token}\nDOCKER_INFLUXDB_INIT_ORG=mesh\nDOCKER_INFLUXDB_INIT_BUCKET=default\n"
},
{
"id": "data",
"type": "directory",
"path": "/services/influxdb/data",
"mode": "0700",
"owner": "1000:1000"
},
{
"id": "config",
"type": "directory",
"path": "/services/influxdb/config",
"mode": "0700",
"owner": "1000:1000"
},
{
"id": "grants",
"type": "directory",
"mode": "0700"
},
{
"id": "server-env",
"type": "file",
"path": "${dir:state}/server.env",
"mode": "0600",
"content": "DOCKER_INFLUXDB_INIT_MODE=setup\nDOCKER_INFLUXDB_INIT_USERNAME=admin\nDOCKER_INFLUXDB_INIT_PASSWORD_FILE=/run/secrets/admin\nDOCKER_INFLUXDB_INIT_ADMIN_TOKEN_FILE=/run/secrets/admin-token\nDOCKER_INFLUXDB_INIT_ORG=mesh\nDOCKER_INFLUXDB_INIT_BUCKET=default\n"
},
{
"id": "server",
"type": "container",
"name": "influxdb",
"image": "influxdb@sha256:f75e48af0598e8aec7986e991a848d19a119101a7d563a2e5db1dfaac9c45daa",
"env-file": [
"${dir:state}/server.env"
"/var/lib/influxdb-module/server.env"
],
"ports": [
"8086"
],
"volumes": [
"${dir:data}:/var/lib/influxdb2",
"${dir:config}:/etc/influxdb2",
"${dir:state}/admin.secret:/run/secrets/admin:ro",
"${dir:state}/admin-token.secret:/run/secrets/admin-token:ro"
]
"/services/influxdb/data:/var/lib/influxdb2",
"/services/influxdb/config:/etc/influxdb2"
],
"secrets-in-environment": "the image honours DOCKER_INFLUXDB_INIT_PASSWORD_FILE and _ADMIN_TOKEN_FILE; convertible, awaiting a bed that proves it"
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/influxdb/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -107,17 +81,15 @@
"name": "mesh-influxdb",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"${dir:state}/admin-token.secret:/run/secrets/admin-token:ro",
"${dir:grants}:${dir:grants}:ro"
"/var/lib/mesh/influxdb/broker:/run/secrets/broker:ro",
"/var/lib/mesh/influxdb/config.json:/run/config/config.json:ro",
"/services/influxdb/config:/var/lib/influxdb/config:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_INFLUXDB_URL": "http://127.0.0.1:${port:8086}",
"MESH_INFLUXDB_URL": "http://127.0.0.1:8086",
"MESH_INFLUXDB_CONFIG_FILE": "/run/config/config.json",
"MESH_INFLUXDB_TOKEN_FILE": "/run/secrets/admin-token",
"MESH_RECEIVES": "${dir:grants}/mesh.json"
"MESH_INFLUXDB_CONFIG_DIR": "/var/lib/influxdb/config"
},
"restart-on": [
"runtime-config"
@@ -125,15 +97,6 @@
"artifact": "runtime"
}
],
"requires": [
"route"
],
"contributes": {
"route": {
"label": "influxdb",
"endpoint": "api"
}
},
"build": {
"on": [
{
@@ -154,5 +117,14 @@
"from": "Dockerfile"
}
]
},
"requires": [
"secret"
],
"secrets": {
"secret": {
"admin": "/var/lib/influxdb-module/admin.secret",
"admin-token": "/var/lib/influxdb-module/admin-token.secret"
}
}
}
+1 -6
View File
@@ -1,14 +1,9 @@
{
"name": "@novox/module-influxdb",
"version": "0.1.0",
"description": "influxdb — time-series database; provides the mesh influxdb-api interface. Its API client, provisioner and tools live here (novox/hq ADR 0039).",
"description": "influxdb — time-series database. Its API client and tools live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"scripts": {
"build": "tsc client.ts grants.ts provisioner/index.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
"typecheck": "tsc -p tsconfig.json",
"test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'"
},
"dependencies": {
"@novox/mesh-sdk": "^0.1.0"
},
-54
View File
@@ -1,54 +0,0 @@
// influxdb's provisioner — the adapter that makes influxdb a provider of the mesh `influxdb-api`
// interface. The reconcile loop, the contributions file and reading the mesh's minted secret are the
// sdk harness's; this writes only the per-service half: how InfluxDB creates, checks and removes a
// consumer's credential (novox/hq ADR 0039/0040/0048). What that credential is, and why it is a v1
// authorization, is in ../grants.ts.
//
// The `influxdb-api` interface: a consumer reaches `${bound:influxdb-api:scheme}://…:at:…:port`,
// signs in as `${bound:influxdb-api:as}` with the password the mesh minted for the pair, and reads
// or writes the org's buckets as databases of the same name — `${bound:influxdb-api:bucket}` being
// the one this instance serves by default. The org and the default bucket are the assignment's
// settings, which reach both what is served and this module's config.json, so the org a consumer is
// told and the org its credential is made in cannot disagree.
import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner";
import { InfluxDBClient } from "../client.js";
import { ApiGrants } from "../grants.js";
let grants: ApiGrants | undefined;
try {
const influx = InfluxDBClient.fromEnv();
grants = new ApiGrants(influx, influx.org);
} catch (err) {
// No admin token: nothing can be provisioned, and the tools loaded beside this must still serve.
console.error(`[provisioner:influxdb-api] not started: ${err instanceof Error ? err.message : err}`);
}
if (grants) serve(grants);
function serve(grants: ApiGrants): void {
runProvisioner("influxdb-api", {
async create(p: Provision): Promise<void> {
const done = await grants.ensure(p);
if (done !== "unchanged") {
console.log(`[provisioner:influxdb-api] ${done} v1 authorization ${p.as} in org ${grants.org}`);
}
},
async remove(p: { as: string }): Promise<void> {
const done = await grants.remove(p.as);
if (done === "not ours") {
console.error(`[provisioner:influxdb-api] ${p.as}: an authorization of that name exists that the mesh did not make — left alone`);
} else if (done === "removed") {
console.log(`[provisioner:influxdb-api] removed v1 authorization ${p.as}; its buckets and their data stay`);
}
},
// Asked every minute by the harness: whether InfluxDB still holds this consumer's authorization
// exactly as the mesh gave it, so one deleted, disabled or re-passworded behind the mesh's back is
// made whole again (hq issue 120).
async holds(p: Provision): Promise<boolean> {
return grants.holds(p);
},
});
}
-246
View File
@@ -1,246 +0,0 @@
// What holds influxdb to the `influxdb-api` provision (grants.ts): one v1 authorization per consumer,
// under the username and password the mesh gave, allowed only what the consumer contributed; made
// once and brought back on every apply; buckets created when missing and never deleted; and an
// authorization the mesh did not make — same name or not — never adopted, changed or deleted.
//
// InfluxDB is a fake: the routes the module touches, answering with the status codes and shapes
// InfluxDB 2.9 gives (a filter matching nothing is a 404, a password outside 8–72 characters a 400,
// an inactive authorization or a wrong password a 401 on /query). Run against the compiled module
// (npm test builds first), the way the runtime loads it.
import { test, after, beforeEach } from "node:test";
import assert from "node:assert/strict";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { InfluxDBClient } from "../dist/client.js";
import { ApiGrants, MARK, askedFor, marked } from "../dist/grants.js";
type Rec = Record<string, any>;
const ADMIN = "operator-token";
const orgs = new Map<string, string>([["zurag", "org1"]]);
let buckets: Rec[] = [];
let auths: Rec[] = [];
let calls: string[] = [];
let seq = 0;
function body(req: IncomingMessage): Promise<any> {
return new Promise((resolve) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => resolve(raw ? JSON.parse(raw) : undefined));
});
}
function send(res: ServerResponse, status: number, value?: unknown): void {
res.writeHead(status, { "Content-Type": "application/json" });
res.end(value === undefined ? "" : JSON.stringify(value));
}
const server = createServer(async (req, res) => {
const url = new URL(req.url!, "http://fake");
const p = url.pathname;
calls.push(`${req.method} ${p}`);
if (p === "/query") {
const basic = (req.headers.authorization ?? "").replace(/^Basic /, "");
const [u, pw] = Buffer.from(basic, "base64").toString().split(":");
const a = auths.find((x) => x.token === u);
if (!a || a.status !== "active" || a.password === undefined || a.password !== pw) {
return send(res, 401, { code: "unauthorized", message: "Unauthorized" });
}
return send(res, 200, { results: [{ statement_id: 0 }] });
}
if (req.headers.authorization !== `Token ${ADMIN}`) return send(res, 401, { code: "unauthorized" });
if (p === "/api/v2/orgs") {
const id = orgs.get(url.searchParams.get("org") ?? "");
if (!id) return send(res, 404, { code: "not found", message: "organization name not found" });
return send(res, 200, { orgs: [{ id, name: url.searchParams.get("org") }] });
}
if (p === "/api/v2/buckets" && req.method === "GET") {
const found = buckets.filter((b) => b.orgID === url.searchParams.get("orgID") && b.name === url.searchParams.get("name"));
if (found.length === 0) return send(res, 404, { code: "not found", message: "bucket not found" });
return send(res, 200, { buckets: found });
}
if (p === "/api/v2/buckets" && req.method === "POST") {
const b = { ...(await body(req)), id: `b${++seq}` };
buckets.push(b);
return send(res, 201, b);
}
if (p === "/private/legacy/authorizations" && req.method === "GET") {
const found = auths.filter((a) => a.token === url.searchParams.get("token"));
if (found.length === 0) return send(res, 404, { code: "not found", message: "authorization not found" });
// Never answers with the password: InfluxDB keeps only its hash.
return send(res, 200, { authorizations: found.map(({ password, ...a }) => ({ ...a, links: {} })) });
}
if (p === "/private/legacy/authorizations" && req.method === "POST") {
const a = await body(req);
if (auths.some((x) => x.token === a.token)) return send(res, 409, { code: "conflict", message: "token already exists" });
const made = { ...a, id: `a${++seq}`, status: a.status ?? "active" };
auths.push(made);
return send(res, 201, made);
}
const m = /^\/private\/legacy\/authorizations\/([^/]+)(\/password)?$/.exec(p);
const a = m && auths.find((x) => x.id === m[1]);
if (!a) return send(res, 404, { code: "not found" });
if (m![2] && req.method === "POST") {
const { password } = await body(req);
if (typeof password !== "string" || password.length < 8 || password.length > 72) {
return send(res, 400, { code: "invalid", message: "passwords must be between 8 and 72 characters long" });
}
a.password = password;
return send(res, 204);
}
if (req.method === "PATCH") {
Object.assign(a, await body(req));
return send(res, 200, a);
}
if (req.method === "DELETE") {
auths = auths.filter((x) => x !== a);
return send(res, 204);
}
send(res, 405);
});
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
after(() => server.close());
const port = (server.address() as { port: number }).port;
const grants = new ApiGrants(new InfluxDBClient(`http://127.0.0.1:${port}`, ADMIN, "zurag"), "zurag");
const PW = "mesh-minted-password-of-forty-characters";
/** Grafana on ace, as the mesh hands it to the provisioner. */
function grafana(password = PW, values: Record<string, unknown> = { access: "read" }) {
return { as: "mesh_ace_grafana", password, consumer: "ace", values };
}
/** Node-RED on ace: writes one bucket. */
function nodered(password = PW, values: Record<string, unknown> = { access: "write", buckets: ["zurag"] }) {
return { as: "mesh_ace_nodered", password, consumer: "ace", values };
}
function only(token: string): Rec {
const found = auths.filter((a) => a.token === token);
assert.equal(found.length, 1, `exactly one authorization ${token}, found ${found.length}`);
return found[0];
}
function perms(a: Rec): string[] {
return a.permissions.map((p: Rec) => `${p.action}:${p.resource.type}:${p.resource.id ?? "*"}`).sort();
}
beforeEach(() => {
buckets = [{ id: "zb", orgID: "org1", name: "zurag" }];
auths = [];
calls = [];
});
test("what a contribution may ask for, and what is refused", () => {
assert.deepEqual(askedFor({}), { access: "read", buckets: [] });
assert.deepEqual(askedFor({ access: "read-write", buckets: ["b", "a", "a"] }), { access: "read-write", buckets: ["a", "b"] });
assert.throws(() => askedFor({ access: "admin" }), /access/);
assert.throws(() => askedFor({ access: "write" }), /names no bucket/);
assert.throws(() => askedFor({ buckets: "zurag" }), /list of bucket names/);
assert.throws(() => askedFor({ access: "write", buckets: ["_monitoring"] }), /system bucket/);
});
test("a reader is given one authorization, reading every bucket of the org, under the mesh's password", async () => {
assert.equal(await grants.ensure(grafana()), "created");
const a = only("mesh_ace_grafana");
assert.equal(a.orgID, "org1");
assert.equal(a.status, "active");
assert.ok(a.description.startsWith(MARK));
assert.deepEqual(perms(a), ["read:buckets:*"]);
assert.equal(a.password, PW);
assert.equal(await grants.holds(grafana()), true);
});
test("a writer is allowed its own buckets only, and a missing one is made — never deleted", async () => {
assert.equal(await grants.ensure(nodered(PW, { access: "write", buckets: ["zurag", "printer"] })), "created");
const made = buckets.find((b) => b.name === "printer");
assert.ok(made, "the missing bucket was created");
assert.deepEqual(made!.retentionRules, [], "kept for ever: retention is the operator's choice");
assert.deepEqual(perms(only("mesh_ace_nodered")), [`write:buckets:${made!.id}`, "write:buckets:zb"]);
assert.equal(await grants.remove("mesh_ace_nodered"), "removed");
assert.equal(buckets.length, 2, "withdrawing the consumer leaves every bucket and its data");
});
test("applying the same grant again writes nothing", async () => {
await grants.ensure(grafana());
calls = [];
assert.equal(await grants.ensure(grafana()), "unchanged");
assert.ok(calls.every((c) => c.startsWith("GET")), `only reads: ${calls.join(", ")}`);
only("mesh_ace_grafana");
});
test("a rotated password is set in place; a changed access remakes only the mesh's own", async () => {
await grants.ensure(nodered());
const id = only("mesh_ace_nodered").id;
assert.equal(await grants.holds(nodered("rotated-password-0123456789")), false);
assert.equal(await grants.ensure(nodered("rotated-password-0123456789")), "updated");
assert.equal(only("mesh_ace_nodered").id, id, "updated, not replaced");
assert.equal(await grants.holds(nodered("rotated-password-0123456789")), true);
await grants.ensure(nodered(PW, { access: "read-write", buckets: ["zurag"] }));
assert.deepEqual(perms(only("mesh_ace_nodered")), ["read:buckets:zb", "write:buckets:zb"]);
assert.equal(await grants.holds(nodered(PW, { access: "read-write", buckets: ["zurag"] })), true);
});
test("an authorization disabled, re-passworded or deleted behind the mesh's back is not held, and is made whole", async () => {
await grants.ensure(grafana());
only("mesh_ace_grafana").status = "inactive";
assert.equal(await grants.holds(grafana()), false);
assert.equal(await grants.ensure(grafana()), "updated");
assert.equal(await grants.holds(grafana()), true);
only("mesh_ace_grafana").password = "somebody-else-set-this";
assert.equal(await grants.holds(grafana()), false);
await grants.ensure(grafana());
assert.equal(await grants.holds(grafana()), true);
auths = [];
assert.equal(await grants.holds(grafana()), false);
assert.equal(await grants.ensure(grafana()), "created");
});
test("holds only reads, and a bucket gone missing is not held rather than made", async () => {
await grants.ensure(nodered());
buckets = [];
calls = [];
assert.equal(await grants.holds(nodered()), false);
assert.ok(calls.every((c) => c.startsWith("GET")), `only reads: ${calls.join(", ")}`);
assert.equal(buckets.length, 0);
});
test("an authorization of the same name the mesh did not make is refused, and left exactly as it was", async () => {
auths = [{ id: "theirs", token: "mesh_ace_grafana", orgID: "org1", status: "active", description: "hand-made",
permissions: [{ action: "write", resource: { type: "buckets", orgID: "org1" } }], password: "their-password" }];
const before = JSON.stringify(auths);
await assert.rejects(grants.ensure(grafana()), /did not make/);
assert.equal(JSON.stringify(auths), before);
assert.ok(calls.every((c) => c.startsWith("GET")), `only reads: ${calls.join(", ")}`);
assert.equal(await grants.holds(grafana()), false);
assert.equal(await grants.remove("mesh_ace_grafana"), "not ours");
assert.equal(auths.length, 1, "never deleted");
});
test("the predecessor's own v1 users and tokens are never touched", async () => {
auths = [{ id: "hal", token: "grafana", orgID: "org1", status: "active", description: "",
permissions: [{ action: "read", resource: { type: "buckets", orgID: "org1" } }], password: "old-password" }];
await grants.ensure(grafana());
assert.equal(auths.find((a) => a.id === "hal")!.password, "old-password");
assert.equal(await grants.remove("grafana"), "not ours");
assert.equal(marked({ token: "grafana", description: `${MARK} x` }), false, "the mark needs the mesh's name too");
});
test("an org the instance does not have, or a non-mesh name, makes nothing", async () => {
const elsewhere = new ApiGrants(new InfluxDBClient(`http://127.0.0.1:${port}`, ADMIN, "nope"), "nope");
await assert.rejects(elsewhere.ensure(grafana()), /no org "nope"/);
await assert.rejects(grants.ensure({ ...grafana(), as: "grafana" }), /not a mesh identity/);
assert.equal(auths.length, 0);
});
test("a withdrawn consumer's authorization is removed, and an absent one is not an error", async () => {
await grants.ensure(grafana());
assert.equal(await grants.remove("mesh_ace_grafana"), "removed");
assert.equal(auths.length, 0);
assert.equal(await grants.remove("mesh_ace_grafana"), "absent");
});
+1 -6
View File
@@ -8,10 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": [
"client.ts",
"grants.ts",
"provisioner/index.ts",
"tools/index.ts"
]
"include": ["client.ts", "tools/index.ts"]
}
+13 -19
View File
@@ -26,13 +26,13 @@
}
},
"binds": {
"mongodb-database": "${dir:state}/database.json",
"s3-bucket": "${dir:state}/store.json",
"route": "${dir:state}/route.json"
"mongodb-database": "/var/lib/invoicing/database.json",
"s3-bucket": "/var/lib/invoicing/store.json",
"route": "/var/lib/invoicing/route.json"
},
"secrets": {
"mongodb-database": "${dir:state}/database.secret",
"s3-bucket": "${dir:state}/store.secret"
"mongodb-database": "/var/lib/invoicing/database.secret",
"s3-bucket": "/var/lib/invoicing/store.secret"
},
"listens": [
{
@@ -54,19 +54,19 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/invoicing",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/invoicing",
"mode": "0700"
},
{
"id": "api-env",
"type": "file",
"path": "${dir:state}/api.env",
"path": "/var/lib/invoicing/api.env",
"mode": "0600",
"content": "NODE_ENV=production\nPORT=9000\nMONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=${bound:mongodb-database:as}\nMONGO_DB=${bound:mongodb-database:as}\nMINIO_BUCKET=mesh-novox-invoice\nMINIO_ENDPOINT=${bound:s3-bucket:at}\nMINIO_PORT=${bound:s3-bucket:port}\nMINIO_ACCESSKEY=${bound:s3-bucket:as}\nMINIO_SECRET=${secret:s3-bucket}\n"
},
@@ -87,10 +87,7 @@
},
"ports": [
"80"
],
"names-on-purpose": {
"registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)"
}
]
},
{
"id": "api",
@@ -103,15 +100,12 @@
"GID": "2201"
},
"env-file": [
"${dir:state}/api.env"
"/var/lib/invoicing/api.env"
],
"ports": [
"9000"
],
"secrets-in-environment": "the application's own code reads MONGO_URL and MINIO_SECRET from the environment (invoicing-app server/src/config.js); converting is that repository's change",
"names-on-purpose": {
"registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)"
}
"secrets-in-environment": "the application's own code reads MONGO_URL and MINIO_SECRET from the environment (invoicing-app server/src/config.js); converting is that repository's change"
}
]
}
+24
View File
@@ -0,0 +1,24 @@
# jackett's runtime: the tool runtime, carrying this module's compiled code.
#
# **Built from this module's own directory and nothing else.** The sdk and the tool runtime are in
# the base images, published like any other artifact — which is what makes this buildable by the
# mesh from a repository and a path (novox/hq ADR 0069) rather than only on a workstation that
# happens to have the siblings.
#
# Two bases, named rather than pinned (novox/hq issue 044): the image this is COMPILED in and the
# image it RUNS in — the second must not carry a compiler. Declared in module.json's `build.on`.
ARG BUILD_BASE
ARG RUNTIME_BASE
FROM ${BUILD_BASE} AS build
WORKDIR /app/modules/jackett
COPY . .
RUN node /app/node_modules/typescript/bin/tsc client.ts tools/index.ts \
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
FROM ${RUNTIME_BASE}
COPY --from=build /app/modules/jackett/dist /app/modules/jackett/dist
# Every serve-time entrypoint, loaded by the runtime in serve mode: tools and events serve, and a
# provider's provisioner runs its reconcile loop in the same process, with the broker connected —
# the convention novox/hq issues 060/061 settled.
ENV MESH_TOOL_MODULES=/app/modules/jackett/dist/tools/index.js
+99
View File
@@ -0,0 +1,99 @@
// The Jackett API client — jackett's own code, living in the module (novox/hq ADR 0039). Jackett is
// an indexer proxy: it normalises many torrent trackers behind one Torznab surface. This client
// talks its /api/v2.0 REST API, and only jackett's tools import it.
import { readFileSync } from "node:fs";
export interface JackettIndexer {
id: string;
name: string;
type: string; // "public" | "private" | "semi-public"
configured: boolean;
siteLink?: string;
lastError?: string;
}
export interface JackettResult {
title: string;
tracker: string;
category?: string;
size: number;
seeders?: number;
peers?: number;
publishDate?: string;
link?: string;
}
/** The settings-merged config the mesh delivers (novox/hq ADR 0046): { url, apiKey, token, password, user, ... }. */
function meshConfig(file?: string): Record<string, string> {
if (!file) return {};
try { return JSON.parse(readFileSync(file, "utf8")) as Record<string, string>; }
catch { return {}; }
}
export class JackettClient {
readonly baseUrl: string;
constructor(
url: string,
private readonly apiKey: string,
) {
this.baseUrl = url.replace(/\/$/, "");
}
/**
* Build from the module's resolved environment. Jackett's REST API is keyed, so both the URL and
* the key must be present — without them there is nothing to talk to, so this throws and the
* module contributes no tools rather than failing half-configured.
*/
static fromEnv(env: NodeJS.ProcessEnv = process.env): JackettClient {
const cfg = meshConfig(env.MESH_JACKETT_CONFIG_FILE);
const url = cfg.url ?? env.MESH_JACKETT_URL;
const apiKey = cfg.apiKey ?? env.MESH_JACKETT_API_KEY;
if (!url) throw new Error("no Jackett URL — set MESH_JACKETT_URL");
if (!apiKey) throw new Error("no Jackett API key — set MESH_JACKETT_API_KEY");
return new JackettClient(url, apiKey);
}
private async get(path: string, params: Record<string, string> = {}): Promise<any> {
const url = new URL(`${this.baseUrl}${path}`);
url.searchParams.set("apikey", this.apiKey);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
const res = await fetch(url.toString(), { headers: { Accept: "application/json" } });
if (!res.ok) throw new Error(`Jackett API ${path}: ${res.status} ${await res.text()}`);
return res.json();
}
/** The configured indexers Jackett proxies. `configured=false` also lists the ones not set up. */
async getIndexers(configuredOnly = true): Promise<JackettIndexer[]> {
const raw = await this.get("/api/v2.0/indexers", { configured: configuredOnly ? "true" : "false" });
const list = Array.isArray(raw) ? raw : [];
return list.map((i: any) => ({
id: i.id,
name: i.name,
type: i.type,
configured: i.configured ?? false,
siteLink: i.site_link,
lastError: i.last_error || undefined,
}));
}
/**
* A Torznab search across one indexer, or the "all" aggregate. Jackett returns a normalised JSON
* result set regardless of the underlying tracker, which is the whole point of the proxy.
*/
async search(query: string, indexer = "all", limit = 25): Promise<JackettResult[]> {
const raw = await this.get(`/api/v2.0/indexers/${encodeURIComponent(indexer)}/results`, { Query: query });
const results = Array.isArray(raw?.Results) ? raw.Results : [];
return results.slice(0, limit).map((r: any) => ({
title: r.Title,
tracker: r.Tracker ?? r.TrackerId ?? "unknown",
category: Array.isArray(r.CategoryDesc) ? r.CategoryDesc.join(", ") : r.CategoryDesc,
size: r.Size ?? 0,
seeders: r.Seeders,
peers: r.Peers,
publishDate: r.PublishDate,
link: r.Link ?? r.Details,
}));
}
}
+113
View File
@@ -0,0 +1,113 @@
{
"module": "jackett",
"version": "1",
"capabilities": [
"container-runtime"
],
"listens": [
{
"name": "web",
"port": 9117,
"protocol": "tcp",
"from": "mesh",
"why": "the indexer proxy"
}
],
"resources": [
{
"id": "mesh-state",
"type": "directory",
"path": "/var/lib/mesh/jackett",
"mode": "0700"
},
{
"id": "config",
"type": "directory",
"path": "/services/jackett/config",
"mode": "0700",
"owner": "1000:1000"
},
{
"id": "server",
"type": "container",
"name": "jackett",
"image": "lscr.io/linuxserver/jackett@sha256:fd72d42b731ebf750b5de9711127251cf3b3f609419c32083ea8b3b3ee840b77",
"env": {
"PUID": "1000",
"PGID": "1000",
"TZ": "Etc/UTC"
},
"ports": [
"9117"
],
"volumes": [
"/services/jackett/config:/config"
]
},
{
"id": "runtime-config",
"type": "file",
"path": "/var/lib/mesh/jackett/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
},
{
"id": "runtime",
"type": "container",
"name": "mesh-jackett",
"network": "host",
"volumes": [
"/var/lib/mesh/jackett/broker:/run/secrets/broker:ro",
"/var/lib/mesh/jackett/config.json:/run/config/config.json:ro",
"/services/jackett/config:/var/lib/jackett/config:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_JACKETT_URL": "http://127.0.0.1:9117",
"MESH_JACKETT_CONFIG_FILE": "/run/config/config.json",
"MESH_JACKETT_CONFIG_DIR": "/var/lib/jackett/config"
},
"restart-on": [
"runtime-config"
],
"artifact": "runtime"
}
],
"own-secrets": {
"broker": "/var/lib/mesh/jackett/broker"
},
"requires": [
"route"
],
"contributes": {
"route": {
"label": "indexers",
"endpoint": "web"
}
},
"binds": {
"route": "/var/lib/mesh/jackett/route.json"
},
"build": {
"on": [
{
"arg": "BUILD_BASE",
"module": "mesh-tools",
"artifact": "build"
},
{
"arg": "RUNTIME_BASE",
"module": "mesh-tools",
"artifact": "runtime"
}
],
"artifacts": [
{
"name": "runtime",
"kind": "image",
"from": "Dockerfile"
}
]
}
}
+14
View File
@@ -0,0 +1,14 @@
{
"name": "@novox/module-jackett",
"version": "0.1.0",
"description": "jackett — indexer proxy. Its API client and tools live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"dependencies": {
"@novox/mesh-sdk": "^0.1.0"
},
"devDependencies": {
"@types/node": "^22.0.0",
"typescript": "^5.6.0"
}
}
+48
View File
@@ -0,0 +1,48 @@
// jackett's tools — its own code (novox/hq ADR 0039), importing jackett's client. Jackett has
// nothing worth watching (an indexer proxy answers queries; it has no timeline of its own), so it
// is a tools-only module: no events entrypoint, no broker. What is useful is asking it things.
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
import { JackettClient } from "../client.js";
export function getJackettTools(jackett: JackettClient): ToolDefinition[] {
return [
{
name: "jackett_indexers",
description: "List the indexers Jackett proxies, with their type and any last error.",
input: { all: { type: "boolean", description: "include indexers not yet configured (default false)" } },
run: async (args) => {
const indexers = await jackett.getIndexers(!args.all);
return { count: indexers.length, indexers };
},
},
{
name: "jackett_search",
description: "Torznab search across Jackett's indexers, returning normalised torrent results.",
input: {
query: { type: "string", description: "the search query" },
indexer: { type: "string", description: 'an indexer id, or "all" to aggregate (default "all")' },
limit: { type: "number", description: "max results (default 25)" },
},
run: async (args) => {
const query = String(args.query);
const results = await jackett.search(
query,
args.indexer ? String(args.indexer) : "all",
args.limit ? Number(args.limit) : 25,
);
return { query, count: results.length, results };
},
},
];
}
// Only exposed when Jackett is configured; otherwise jackett contributes no tools rather than
// failing the whole runtime.
registerModuleTools("jackett", (env) => {
try {
return getJackettTools(JackettClient.fromEnv(env));
} catch {
return [];
}
});
@@ -8,5 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": ["records.ts", "index.ts", "tools/index.ts"]
"include": ["client.ts", "tools/index.ts"]
}
+10 -10
View File
@@ -2,26 +2,26 @@
"module": "jira",
"version": "1",
"own-secrets": {
"token": "${dir:state}/token",
"broker": "${dir:mesh-state}/broker"
"token": "/var/lib/jira/token",
"broker": "/var/lib/mesh/jira/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/jira",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/jira",
"mode": "0700"
},
{
"id": "config",
"type": "file",
"path": "${dir:state}/config.json",
"path": "/var/lib/jira/config.json",
"merge": "json",
"content": "{}",
"mode": "0600"
@@ -32,9 +32,9 @@
"name": "mesh-runtime-jira",
"network": "host",
"volumes": [
"${dir:state}/config.json:/run/config/config.json:ro",
"${dir:state}/token:/run/secrets/token:ro",
"${dir:mesh-state}/broker:/run/secrets/broker:ro"
"/var/lib/jira/config.json:/run/config/config.json:ro",
"/var/lib/jira/token:/run/secrets/token:ro",
"/var/lib/mesh/jira/broker:/run/secrets/broker:ro"
],
"env": {
"MESH_JIRA_TOKEN_FILE": "/run/secrets/token",
+2 -2
View File
@@ -17,7 +17,7 @@ FROM ${BUILD_BASE} AS build
# resolved away.
WORKDIR /app/modules/keycloak
COPY . .
RUN node /app/node_modules/typescript/bin/tsc client.ts oidc.ts index.ts provisioner/index.ts tools/index.ts \
RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts tools/index.ts \
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
FROM ${RUNTIME_BASE}
@@ -27,4 +27,4 @@ COPY --from=build /app/modules/keycloak/dist /app/modules/keycloak/dist
# the convention novox/hq issues 060/061 settled. A container that instead ran only its
# provisioner (`run`) served no tools and emitted no events; a container that named no command
# ran no provisioner at all.
ENV MESH_TOOL_MODULES=/app/modules/keycloak/dist/index.js,/app/modules/keycloak/dist/tools/index.js,/app/modules/keycloak/dist/provisioner/index.js
ENV MESH_TOOL_MODULES=/app/modules/keycloak/dist/index.js,/app/modules/keycloak/dist/tools/index.js
+2 -90
View File
@@ -12,45 +12,6 @@ function meshConfig(file?: string): Record<string, string> {
catch { return {}; }
}
/** A secret file's value, trailing newline trimmed; undefined when unset or unreadable. */
function secretFile(file?: string): string | undefined {
if (!file) return undefined;
try { return readFileSync(file, "utf8").replace(/\n$/, "") || undefined; }
catch { return undefined; }
}
/** A client as the admin API represents it — only the fields this module reads or writes are typed;
* the rest travel through untouched, so an update never drops what somebody else set. */
export interface ClientRepresentation {
id?: string;
clientId: string;
name?: string;
enabled?: boolean;
protocol?: string;
publicClient?: boolean;
clientAuthenticatorType?: string;
secret?: string;
rootUrl?: string;
baseUrl?: string;
redirectUris?: string[];
webOrigins?: string[];
standardFlowEnabled?: boolean;
implicitFlowEnabled?: boolean;
directAccessGrantsEnabled?: boolean;
serviceAccountsEnabled?: boolean;
attributes?: Record<string, string>;
protocolMappers?: ProtocolMapperRepresentation[];
[other: string]: unknown;
}
export interface ProtocolMapperRepresentation {
id?: string;
name: string;
protocol: string;
protocolMapper: string;
config: Record<string, string>;
}
export class KeycloakClient {
readonly baseUrl: string;
readonly defaultRealm: string;
@@ -79,13 +40,8 @@ export class KeycloakClient {
const cfg = meshConfig(env.MESH_KEYCLOAK_CONFIG_FILE);
const url = cfg.url ?? env.MESH_KEYCLOAK_URL ?? `http://127.0.0.1:${env.KEYCLOAK_PORT ?? "8080"}`;
const adminUser = cfg.user ?? env.MESH_KEYCLOAK_ADMIN ?? env.KEYCLOAK_ADMIN ?? "admin";
// The admin password reaches the runtime as a file (novox/hq ADR 0086): the module's own `admin`
// secret, mounted read-only. The environment forms stay for a co-located server that has them.
const adminPass = cfg.password ?? secretFile(env.MESH_KEYCLOAK_PASSWORD_FILE)
?? env.MESH_KEYCLOAK_PASSWORD ?? env.KEYCLOAK_ADMIN_PASSWORD;
if (!adminPass) {
throw new Error("no Keycloak admin password — set MESH_KEYCLOAK_PASSWORD_FILE (or MESH_KEYCLOAK_PASSWORD)");
}
const adminPass = cfg.password ?? env.MESH_KEYCLOAK_PASSWORD ?? env.KEYCLOAK_ADMIN_PASSWORD;
if (!adminPass) throw new Error("no Keycloak admin password — set MESH_KEYCLOAK_PASSWORD");
const realm = cfg.realm ?? env.MESH_KEYCLOAK_REALM ?? "master";
return new KeycloakClient(url, adminUser, adminPass, realm);
}
@@ -203,50 +159,6 @@ export class KeycloakClient {
return client.id as string;
}
/** The one client with exactly this clientId, or undefined. The admin API's `clientId` filter is an
* exact match unless `search=true` is asked for. */
async findClient(realm: string, clientId: string): Promise<ClientRepresentation | undefined> {
const found = await this.request<ClientRepresentation[]>(
`/${realm}/clients?clientId=${encodeURIComponent(clientId)}`);
return found.find((c) => c.clientId === clientId);
}
async createClientFrom(realm: string, rep: ClientRepresentation): Promise<void> {
await this.request(`/${realm}/clients`, { method: "POST", body: JSON.stringify(rep) });
}
/** Replace a client's representation, addressed by its internal id. */
async updateClient(realm: string, id: string, rep: ClientRepresentation): Promise<void> {
await this.request(`/${realm}/clients/${id}`, { method: "PUT", body: JSON.stringify(rep) });
}
async deleteClientById(realm: string, id: string): Promise<void> {
await this.request(`/${realm}/clients/${id}`, { method: "DELETE" });
}
async clientSecretById(realm: string, id: string): Promise<string | undefined> {
const result = await this.request<{ value?: string }>(`/${realm}/clients/${id}/client-secret`);
return result.value;
}
async listClientMappers(realm: string, id: string): Promise<ProtocolMapperRepresentation[]> {
return this.request(`/${realm}/clients/${id}/protocol-mappers/models`);
}
async addClientMapper(realm: string, id: string, mapper: ProtocolMapperRepresentation): Promise<void> {
await this.request(`/${realm}/clients/${id}/protocol-mappers/models`, {
method: "POST",
body: JSON.stringify(mapper),
});
}
async updateClientMapper(realm: string, id: string, mapper: ProtocolMapperRepresentation): Promise<void> {
await this.request(`/${realm}/clients/${id}/protocol-mappers/models/${mapper.id}`, {
method: "PUT",
body: JSON.stringify(mapper),
});
}
async deleteClient(realm: string, clientId: string): Promise<void> {
await this.request(`/${realm}/clients/${await this.resolveClientId(realm, clientId)}`, { method: "DELETE" });
}
+18 -57
View File
@@ -1,12 +1,6 @@
{
"module": "keycloak",
"version": "1",
"provides": [
{
"name": "oidc-client",
"scope": "mesh"
}
],
"requires": [
"postgres-database",
"route"
@@ -21,11 +15,11 @@
}
},
"binds": {
"postgres-database": "${dir:state}/database.json",
"route": "${dir:state}/route.json"
"postgres-database": "/var/lib/keycloak/database.json",
"route": "/var/lib/keycloak/route.json"
},
"secrets": {
"postgres-database": "${dir:state}/database.secret"
"postgres-database": "/var/lib/keycloak/database.secret"
},
"capabilities": [
"container-runtime"
@@ -47,53 +41,34 @@
"why": "anything the mesh runs that authenticates a person"
}
],
"serves": {
"oidc-client": {
"authorization-path": "/protocol/openid-connect/auth",
"token-path": "/protocol/openid-connect/token",
"userinfo-path": "/protocol/openid-connect/userinfo",
"issuer": "${setting:issuer}"
}
},
"receives": {
"oidc-client": "${dir:grants}/mesh.json"
},
"grants": {
"oidc-client": "${dir:grants}"
},
"own-secrets": {
"admin": "${dir:state}/admin.secret",
"broker": "${dir:mesh-state}/broker"
"admin": "/var/lib/keycloak/admin.secret",
"broker": "/var/lib/mesh/keycloak/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/keycloak",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
},
{
"id": "grants",
"type": "directory",
"path": "/var/lib/keycloak",
"mode": "0700"
},
{
"id": "admin-env",
"type": "file",
"path": "${dir:state}/admin.env",
"path": "/var/lib/keycloak/admin.env",
"mode": "0600",
"content": "KEYCLOAK_ADMIN=admin\nKEYCLOAK_ADMIN_PASSWORD=${secret:admin}\n"
},
{
"id": "database-env",
"type": "file",
"path": "${dir:state}/database.env",
"path": "/var/lib/keycloak/database.env",
"mode": "0600",
"content": "KC_DB_URL=jdbc:postgresql://${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\nKC_DB_USERNAME=${bound:postgres-database:as}\nKC_DB_PASSWORD=${secret:postgres-database}\n"
},
@@ -102,13 +77,6 @@
"type": "network",
"name": "keycloak"
},
{
"id": "hostname",
"type": "file",
"path": "${dir:state}/hostname.env",
"mode": "0644",
"content": "KC_HOSTNAME=https://${bound:route:name}\n"
},
{
"id": "server",
"type": "container",
@@ -122,25 +90,22 @@
"KC_DB": "postgres",
"KC_HTTP_ENABLED": "true",
"KC_HEALTH_ENABLED": "true",
"KC_HOSTNAME": "https://keycloak.novox.be",
"KC_PROXY_HEADERS": "xforwarded"
},
"env-file": [
"${dir:state}/admin.env",
"${dir:state}/database.env",
"${dir:state}/hostname.env"
"/var/lib/keycloak/admin.env",
"/var/lib/keycloak/database.env"
],
"ports": [
"8080"
],
"secrets-in-environment": "KC_DB_PASSWORD is convertible through a generated keycloak.conf (db-password=); KEYCLOAK_ADMIN_PASSWORD is env-only before Keycloak 26; not yet converted",
"restart-on": [
"hostname"
]
"secrets-in-environment": "KC_DB_PASSWORD is convertible through a generated keycloak.conf (db-password=); KEYCLOAK_ADMIN_PASSWORD is env-only before Keycloak 26; not yet converted"
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/keycloak/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -151,17 +116,13 @@
"name": "mesh-keycloak",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"${dir:state}/admin.secret:/run/secrets/admin:ro",
"${dir:grants}:${dir:grants}:ro"
"/var/lib/mesh/keycloak/broker:/run/secrets/broker:ro",
"/var/lib/mesh/keycloak/config.json:/run/config/config.json:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_KEYCLOAK_URL": "http://127.0.0.1:${port:8080}",
"MESH_KEYCLOAK_CONFIG_FILE": "/run/config/config.json",
"MESH_KEYCLOAK_PASSWORD_FILE": "/run/secrets/admin",
"MESH_RECEIVES": "${dir:grants}/mesh.json"
"MESH_KEYCLOAK_CONFIG_FILE": "/run/config/config.json"
},
"restart-on": [
"runtime-config"
-185
View File
@@ -1,185 +0,0 @@
// What the `oidc-client` provision means in Keycloak: one confidential OpenID Connect client per
// consumer, in the realm this module serves, under the name and secret the mesh gave both ends.
// The provisioner (provisioner/index.ts) is the sdk harness calling these; they are here, apart from
// it, so they can be exercised against a fake admin API without a broker or a contributions file.
//
// **The client id and the secret are the mesh's, not Keycloak's (novox/hq ADR 0048).** The mesh
// derives the consumer's identity (`as`, e.g. `mesh_ace_grafana`) and hands it to both ends — the
// consumer names it as its client id through `${bound:oidc-client:as}` — and mints the secret, which
// this sets as the client's secret. Keycloak generates neither.
//
// **Where the consumer's browser comes back to is the consumer's to say.** Its contribution carries
// `callback` (a path, e.g. `/login/generic_oauth`) and the `label`/`endpoint` of the endpoint it is
// reached on; the mesh composes that endpoint's names into `name` (public) and `internal-name`
// (private network) exactly as it does for a route (novox/hq ADR 0056, 0138), so the redirect URI
// registered here is built from the same names the proxy serves the consumer under.
//
// **Only what the mesh made is touched.** A client this module creates carries the attribute
// `mesh.provisioned=true`, and its id starts with the mesh's own prefix. A client with the same id
// that lacks the mark is somebody else's: it is refused, never adopted, never updated, never deleted.
import type { ClientRepresentation, KeycloakClient, ProtocolMapperRepresentation } from "./client.js";
/** The attribute marking a client as the mesh's own work. */
export const MARK = "mesh.provisioned";
/** The mapper every mesh client carries: realm roles as a flat `roles` claim in the id token, the
* access token and userinfo — what a consumer maps its own roles from (grafana's role path reads
* `roles[*]`), and what the predecessor added to its hand-made clients by hand. */
export const ROLES_MAPPER: ProtocolMapperRepresentation = {
name: "realm roles",
protocol: "openid-connect",
protocolMapper: "oidc-usermodel-realm-role-mapper",
config: {
"claim.name": "roles",
"jsonType.label": "String",
multivalued: "true",
"id.token.claim": "true",
"access.token.claim": "true",
"userinfo.token.claim": "true",
},
};
/** One consumer, as the harness hands it over. */
export interface OidcGrant {
readonly as: string;
readonly password: string;
readonly values: Readonly<Record<string, unknown>>;
readonly consumer?: string;
}
/** The realm named by an issuer URL — `https://id.example/realms/Novox` is realm `Novox`. The issuer is
* the one value an assignment sets (it is also what consumers are served), so the realm is read
* out of it rather than set a second time where the two could disagree. */
export function realmOf(issuer: string): string {
let path: string;
try {
path = new URL(issuer).pathname;
} catch {
throw new Error(`the issuer ${JSON.stringify(issuer)} is not a URL`);
}
const m = /\/realms\/([^/]+)\/?$/.exec(path);
if (!m) throw new Error(`the issuer ${JSON.stringify(issuer)} does not end in /realms/<realm>`);
return decodeURIComponent(m[1]);
}
/** The redirect URIs a consumer's contribution asks for: its callback under each name the mesh
* composed for its endpoint. Refused when there is nothing to register — a client that accepts no
* redirect is a client nobody can log in through, and one that accepts any is worse. */
export function redirectsOf(values: Readonly<Record<string, unknown>>): { root: string; redirects: string[] } {
const callback = values.callback;
if (typeof callback !== "string" || !callback.startsWith("/")) {
throw new Error(`contributes no callback path (\`callback\`, starting with "/"): ${JSON.stringify(callback)}`);
}
const names: string[] = [];
for (const key of ["name", "internal-name"]) {
const n = values[key];
if (typeof n === "string" && n.trim() !== "" && !names.includes(n.trim())) names.push(n.trim());
}
if (names.length === 0) {
throw new Error("has no name the mesh composed (`name` / `internal-name`) — contribute a `label` and the `endpoint` it is reached on");
}
return { root: `https://${names[0]}`, redirects: names.map((n) => `https://${n}${callback}`) };
}
/** The fields the mesh owns on a client it made. Everything else on the client is left as found. */
function wanted(g: OidcGrant): ClientRepresentation {
const { root, redirects } = redirectsOf(g.values);
return {
clientId: g.as,
name: g.as,
description: `made by the mesh for ${g.consumer ? `a module on ${g.consumer}` : "a consumer"} — do not edit; it is reset`,
enabled: true,
protocol: "openid-connect",
publicClient: false,
clientAuthenticatorType: "client-secret",
secret: g.password,
rootUrl: root,
baseUrl: root,
redirectUris: redirects,
standardFlowEnabled: true,
implicitFlowEnabled: false,
directAccessGrantsEnabled: false,
serviceAccountsEnabled: false,
};
}
function sameSet(a: readonly string[] | undefined, b: readonly string[]): boolean {
const x = [...(a ?? [])].sort();
const y = [...b].sort();
return x.length === y.length && x.every((v, i) => v === y[i]);
}
function marked(c: ClientRepresentation): boolean {
return c.attributes?.[MARK] === "true";
}
export class OidcClients {
constructor(private readonly kc: KeycloakClient, readonly realm: string) {}
/** Create the consumer's client, or bring the mesh's existing one back to what the grant says.
* Returns whether it was newly created. Idempotent: applying the same grant twice changes nothing
* the second time beyond re-asserting it. */
async ensure(g: OidcGrant): Promise<"created" | "updated"> {
const want = wanted(g);
const found = await this.kc.findClient(this.realm, g.as);
if (found && !marked(found)) {
throw new Error(
`realm ${this.realm} already has a client ${g.as} the mesh did not make — left alone; ` +
`delete or rename it if the mesh should own that id`);
}
if (!found) {
await this.kc.createClientFrom(this.realm, {
...want,
attributes: { [MARK]: "true" },
protocolMappers: [ROLES_MAPPER],
});
return "created";
}
// Overlay what the mesh owns on what is there, so a field Keycloak added or an operator set on a
// field the mesh does not own survives the update.
await this.kc.updateClient(this.realm, found.id!, {
...found,
...want,
attributes: { ...(found.attributes ?? {}), [MARK]: "true" },
});
await this.ensureMapper(found.id!);
return "updated";
}
private async ensureMapper(id: string): Promise<void> {
const mappers = await this.kc.listClientMappers(this.realm, id);
const have = mappers.find((m) => m.name === ROLES_MAPPER.name);
if (!have) {
await this.kc.addClientMapper(this.realm, id, ROLES_MAPPER);
return;
}
const drifted =
have.protocolMapper !== ROLES_MAPPER.protocolMapper ||
Object.entries(ROLES_MAPPER.config).some(([k, v]) => have.config?.[k] !== v);
if (drifted) {
await this.kc.updateClientMapper(this.realm, id, { ...ROLES_MAPPER, id: have.id });
}
}
/** Whether Keycloak still holds this consumer's client exactly as the grant says: present, the
* mesh's, enabled, confidential, with the mesh's secret and the redirects asked for. Reads only. */
async holds(g: OidcGrant): Promise<boolean> {
const want = wanted(g);
const found = await this.kc.findClient(this.realm, g.as);
if (!found || !marked(found) || found.enabled === false || found.publicClient) return false;
if (!sameSet(found.redirectUris, want.redirectUris!)) return false;
const mappers = await this.kc.listClientMappers(this.realm, found.id!);
if (!mappers.some((m) => m.name === ROLES_MAPPER.name)) return false;
return (await this.kc.clientSecretById(this.realm, found.id!)) === g.password;
}
/** Withdraw a consumer's client — only one the mesh made. Returns what happened, for the log. */
async remove(as: string): Promise<"removed" | "absent" | "not ours"> {
const found = await this.kc.findClient(this.realm, as);
if (!found) return "absent";
if (!marked(found)) return "not ours";
await this.kc.deleteClientById(this.realm, found.id!);
return "removed";
}
}
+2 -6
View File
@@ -1,15 +1,11 @@
{
"name": "@novox/module-keycloak",
"version": "0.1.0",
"description": "keycloak — identity and access; provides the mesh oidc-client interface. Its admin API client, provisioner, tools and events live here (novox/hq ADR 0039).",
"description": "keycloak — identity and access. Its admin API client, tools and events live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"scripts": {
"build": "tsc client.ts oidc.ts index.ts provisioner/index.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
"test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'"
},
"dependencies": {
"@novox/mesh-sdk": "^0.1.1"
"@novox/mesh-sdk": "^0.1.0"
},
"devDependencies": {
"@types/node": "^22.0.0",
-73
View File
@@ -1,73 +0,0 @@
// keycloak's provisioner — the adapter that makes keycloak a provider of the mesh `oidc-client`
// interface. The reconcile loop, the contributions file and reading the mesh's minted secret are the
// sdk harness's; this writes only the per-service half: how Keycloak creates, checks and removes a
// consumer's client (novox/hq ADR 0039/0040/0048). What a client is, and which ones are the mesh's,
// is in ../oidc.ts.
//
// The `oidc-client` interface: a consumer logs people in through the realm this module serves, as
// the confidential client `as` with the secret the mesh minted, and is redirected back to the
// callback it contributed under the names the mesh composed for its endpoint. What it is served —
// the issuer and the endpoint paths under it — is in the manifest's `serves`, settled with the
// assignment's settings.
//
// **The realm is read out of the issuer**, the one value an assignment sets (settings reach both the
// served facts and this module's config.json): a realm set in one place and an issuer in another
// would let the consumer be told one realm while its client is made in another.
import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner";
import { emit } from "@novox/mesh-sdk/events";
import { readFileSync } from "node:fs";
import { KeycloakClient } from "../client.js";
import { OidcClients, realmOf } from "../oidc.js";
/** The issuer this assignment serves, from the settings-merged config the mesh delivers. */
function issuer(): string {
const file = process.env.MESH_KEYCLOAK_CONFIG_FILE;
let cfg: Record<string, unknown> = {};
if (file) {
try {
cfg = JSON.parse(readFileSync(file, "utf8")) as Record<string, unknown>;
} catch {
// Absent or unreadable: fall through to the environment, and refuse below if that is empty too.
}
}
const said = typeof cfg.issuer === "string" ? cfg.issuer : process.env.MESH_KEYCLOAK_ISSUER;
if (!said) throw new Error("no issuer — the module's config.json carries none and MESH_KEYCLOAK_ISSUER is unset");
return said;
}
const clients = new OidcClients(KeycloakClient.fromEnv(), realmOf(issuer()));
/** Emit a lifecycle event without letting a broker hiccup fail the provisioning itself. */
async function announce(type: string, body: Record<string, string>): Promise<void> {
try {
await emit(type, body);
} catch (err) {
console.error(`[provisioner:oidc-client] emit ${type} failed: ${err}`);
}
}
runProvisioner("oidc-client", {
async create(p: Provision): Promise<void> {
const done = await clients.ensure(p);
if (done === "created") {
console.log(`[provisioner:oidc-client] created client ${p.as} in realm ${clients.realm}`);
await announce("client.created", { realm: clients.realm, clientId: p.as, consumer: p.consumer ?? "" });
}
},
async remove(p: { as: string }): Promise<void> {
const done = await clients.remove(p.as);
if (done === "not ours") {
console.error(`[provisioner:oidc-client] ${p.as}: a client of that id exists that the mesh did not make — left alone`);
} else if (done === "removed") {
console.log(`[provisioner:oidc-client] removed client ${p.as} from realm ${clients.realm}`);
}
},
// Asked every minute by the harness: whether Keycloak still holds this consumer's client exactly as
// the mesh gave it, so a client deleted or edited behind the mesh's back is made again (hq issue 120).
async holds(p: Provision): Promise<boolean> {
return clients.holds(p);
},
});
-239
View File
@@ -1,239 +0,0 @@
// What holds keycloak to the `oidc-client` provision (oidc.ts): one confidential client per consumer,
// under the id and secret the mesh gave, redirecting only to the consumer's own callback under the
// names the mesh composed; made once and brought back on every apply; and a client the mesh did not
// make — same id or not — never adopted, changed or deleted.
//
// Keycloak is a fake: the admin routes the module touches, answering with the status codes and the
// shapes Keycloak gives. Run against the compiled module (npm test builds first), the way the runtime
// loads it.
import { test, after } from "node:test";
import assert from "node:assert/strict";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { randomUUID } from "node:crypto";
import { KeycloakClient } from "../dist/client.js";
import { MARK, OidcClients, ROLES_MAPPER, realmOf, redirectsOf } from "../dist/oidc.js";
type Client = Record<string, any>;
/** The realm's clients, by internal id, and what the fake was asked. */
const realm = "Novox";
const clients = new Map<string, Client>();
const calls: string[] = [];
function body(req: IncomingMessage): Promise<any> {
return new Promise((resolve) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => resolve(raw ? JSON.parse(raw) : undefined));
});
}
function send(res: ServerResponse, status: number, value?: unknown): void {
res.writeHead(status, { "Content-Type": "application/json" });
res.end(value === undefined ? "" : JSON.stringify(value));
}
const server = createServer(async (req, res) => {
const url = new URL(req.url!, "http://fake");
calls.push(`${req.method} ${url.pathname}`);
if (url.pathname === "/realms/master/protocol/openid-connect/token") {
return send(res, 200, { access_token: "t", expires_in: 300 });
}
const base = `/admin/realms/${realm}/clients`;
if (!url.pathname.startsWith(base)) return send(res, 404, { error: "Realm not found." });
const rest = url.pathname.slice(base.length).split("/").filter(Boolean);
if (rest.length === 0 && req.method === "GET") {
const want = url.searchParams.get("clientId");
return send(res, 200, [...clients.values()].filter((c) => !want || c.clientId === want));
}
if (rest.length === 0 && req.method === "POST") {
const rep = await body(req);
if ([...clients.values()].some((c) => c.clientId === rep.clientId)) {
return send(res, 409, { errorMessage: `Client ${rep.clientId} already exists` });
}
const id = randomUUID();
const mappers = (rep.protocolMappers ?? []).map((m: Client) => ({ ...m, id: randomUUID() }));
clients.set(id, { ...rep, id, protocolMappers: mappers });
return send(res, 201);
}
const c = clients.get(rest[0]);
if (!c) return send(res, 404, { error: "Could not find client" });
if (rest.length === 1 && req.method === "PUT") {
// Keycloak ignores protocolMappers on a client update: they have their own endpoints.
const rep = await body(req);
clients.set(c.id, { ...rep, id: c.id, protocolMappers: c.protocolMappers });
return send(res, 204);
}
if (rest.length === 1 && req.method === "DELETE") {
clients.delete(c.id);
return send(res, 204);
}
if (rest[1] === "client-secret" && req.method === "GET") {
return send(res, 200, { type: "secret", value: c.secret });
}
if (rest[1] === "protocol-mappers") {
if (req.method === "GET") return send(res, 200, c.protocolMappers ?? []);
if (req.method === "POST") {
c.protocolMappers = [...(c.protocolMappers ?? []), { ...(await body(req)), id: randomUUID() }];
return send(res, 201);
}
if (req.method === "PUT") {
const m = await body(req);
c.protocolMappers = c.protocolMappers.map((x: Client) => (x.id === rest[4] ? m : x));
return send(res, 204);
}
}
send(res, 405);
});
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
after(() => server.close());
const port = (server.address() as { port: number }).port;
const oidc = new OidcClients(new KeycloakClient(`http://127.0.0.1:${port}`, "admin", "pw"), realm);
/** Grafana on ace, as the mesh hands it to the provisioner. */
function grafana(secret = "s3cret", values: Record<string, unknown> = {}) {
return {
as: "mesh_ace_grafana",
password: secret,
consumer: "ace",
values: {
label: "grafana", endpoint: "web", port: 20010, callback: "/login/generic_oauth",
name: "grafana.zurag.be", "internal-name": "grafana.ace.internal", ...values,
},
};
}
function only(clientId: string): Client {
const found = [...clients.values()].filter((c) => c.clientId === clientId);
assert.equal(found.length, 1, `exactly one client ${clientId}, found ${found.length}`);
return found[0];
}
test("the realm is read out of the issuer, and an issuer that names none is refused", () => {
assert.equal(realmOf("https://keycloak.novox.be/realms/Novox"), "Novox");
assert.equal(realmOf("https://keycloak.novox.be/realms/Novox/"), "Novox");
assert.equal(realmOf("http://127.0.0.1:18500/realms/master"), "master");
assert.throws(() => realmOf("https://keycloak.novox.be"), /realms/);
assert.throws(() => realmOf("keycloak"), /not a URL/);
});
test("the redirect is the consumer's callback under every name the mesh composed for it", () => {
assert.deepEqual(redirectsOf(grafana().values), {
root: "https://grafana.zurag.be",
redirects: ["https://grafana.zurag.be/login/generic_oauth", "https://grafana.ace.internal/login/generic_oauth"],
});
// A route reaching only the private network has only the internal name, and that is enough.
assert.deepEqual(redirectsOf({ callback: "/cb", "internal-name": "x.ace.internal" }).redirects,
["https://x.ace.internal/cb"]);
assert.throws(() => redirectsOf({ name: "grafana.zurag.be" }), /callback/);
assert.throws(() => redirectsOf({ name: "grafana.zurag.be", callback: "login" }), /callback/);
assert.throws(() => redirectsOf({ callback: "/cb" }), /label/);
});
test("a consumer is given one confidential client, under its id and the mesh's secret", async () => {
clients.clear();
assert.equal(await oidc.ensure(grafana()), "created");
const c = only("mesh_ace_grafana");
assert.equal(c.publicClient, false);
assert.equal(c.clientAuthenticatorType, "client-secret");
assert.equal(c.secret, "s3cret");
assert.equal(c.enabled, true);
assert.equal(c.standardFlowEnabled, true);
assert.equal(c.directAccessGrantsEnabled, false);
assert.equal(c.implicitFlowEnabled, false);
assert.deepEqual(c.redirectUris, [
"https://grafana.zurag.be/login/generic_oauth", "https://grafana.ace.internal/login/generic_oauth"]);
assert.equal(c.attributes[MARK], "true");
assert.deepEqual(c.protocolMappers.map((m: Client) => m.name), [ROLES_MAPPER.name]);
assert.equal(await oidc.holds(grafana()), true);
});
test("applying the same grant again makes no second client", async () => {
clients.clear();
await oidc.ensure(grafana());
assert.equal(await oidc.ensure(grafana()), "updated");
assert.equal(await oidc.ensure(grafana()), "updated");
only("mesh_ace_grafana");
assert.equal(only("mesh_ace_grafana").protocolMappers.length, 1, "the roles mapper is not added twice");
});
test("a new secret or a moved name is applied in place, and what the mesh does not own survives", async () => {
clients.clear();
await oidc.ensure(grafana());
const id = only("mesh_ace_grafana").id;
// Something the mesh does not own, set on the client after it was made.
clients.get(id)!.consentRequired = true;
clients.get(id)!.attributes["post.logout.redirect.uris"] = "+";
assert.equal(await oidc.holds(grafana("rotated")), false, "a rotated secret is not held until applied");
await oidc.ensure(grafana("rotated", { name: "dash.zurag.be" }));
const c = only("mesh_ace_grafana");
assert.equal(c.id, id, "updated, not replaced");
assert.equal(c.secret, "rotated");
assert.deepEqual(c.redirectUris, [
"https://dash.zurag.be/login/generic_oauth", "https://grafana.ace.internal/login/generic_oauth"]);
assert.equal(c.rootUrl, "https://dash.zurag.be");
assert.equal(c.consentRequired, true);
assert.equal(c.attributes["post.logout.redirect.uris"], "+");
assert.equal(c.attributes[MARK], "true");
assert.equal(await oidc.holds(grafana("rotated", { name: "dash.zurag.be" })), true);
});
test("a client lost or edited behind the mesh's back is not held, and is made whole again", async () => {
clients.clear();
await oidc.ensure(grafana());
const c = only("mesh_ace_grafana");
c.redirectUris = ["*"];
assert.equal(await oidc.holds(grafana()), false, "a widened redirect is not what the mesh gave");
await oidc.ensure(grafana());
assert.equal(await oidc.holds(grafana()), true);
only("mesh_ace_grafana").protocolMappers = [];
assert.equal(await oidc.holds(grafana()), false, "a client without its roles mapper is not held");
await oidc.ensure(grafana());
assert.equal(await oidc.holds(grafana()), true);
clients.clear();
assert.equal(await oidc.holds(grafana()), false);
});
test("a client of the same id the mesh did not make is refused, and left exactly as it was", async () => {
clients.clear();
clients.set("theirs", { id: "theirs", clientId: "mesh_ace_grafana", secret: "their-secret", redirectUris: ["*"] });
const before = JSON.stringify(clients.get("theirs"));
const writes = calls.length;
await assert.rejects(oidc.ensure(grafana()), /did not make/);
assert.equal(JSON.stringify(clients.get("theirs")), before);
assert.ok(calls.slice(writes).every((c) => c.startsWith("GET") || c.startsWith("POST /realms/master")),
`only reads were made: ${calls.slice(writes).join(", ")}`);
assert.equal(await oidc.holds(grafana()), false);
assert.equal(await oidc.remove("mesh_ace_grafana"), "not ours");
assert.ok(clients.has("theirs"), "a client the mesh did not make is never deleted");
});
test("the predecessor's hand-made client is never touched: the mesh's has its own id", async () => {
clients.clear();
clients.set("hal", { id: "hal", clientId: "grafana", secret: "old", redirectUris: ["https://grafana.zurag.be/*"] });
await oidc.ensure(grafana());
assert.equal(clients.get("hal")!.secret, "old");
only("mesh_ace_grafana");
assert.equal(await oidc.remove("grafana"), "not ours");
assert.ok(clients.has("hal"));
});
test("a withdrawn consumer's client is removed, and an absent one is not an error", async () => {
clients.clear();
await oidc.ensure(grafana());
assert.equal(await oidc.remove("mesh_ace_grafana"), "removed");
assert.equal([...clients.values()].length, 0);
assert.equal(await oidc.remove("mesh_ace_grafana"), "absent");
});
test("a contribution with no callback makes no client at all", async () => {
clients.clear();
await assert.rejects(oidc.ensure({ ...grafana(), values: { name: "grafana.zurag.be" } }), /callback/);
assert.equal(clients.size, 0);
});
+1 -1
View File
@@ -8,5 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": ["client.ts", "oidc.ts", "index.ts", "provisioner/index.ts", "tools/index.ts"]
"include": ["client.ts", "index.ts", "tools/index.ts"]
}
+4 -8
View File
@@ -1,10 +1,10 @@
// The Letta API client — letta's own code, living in the module (novox/hq ADR 0039). Its tools
// import it; nothing outside letta does.
//
// Letta authenticates with a single server password. That password is a mesh own-secret handed to
// both the server (LETTA_SERVER_PASSWORD) and this client, through the runtime config file the mesh
// mounts (its `password` key) — so the module's tools are live without anything configured by hand.
// Where a server already has clients, the password is accepted rather than minted.
// Letta authenticates with a single server password, presented as a Bearer token. That password is
// a mesh own-secret, minted once and handed to both the server (LETTA_SERVER_PASSWORD) and this
// client (MESH_LETTA_PASSWORD) — so the module's tools are live without anything configured by hand.
// The runtime config file may still override the URL or password.
import { readFileSync } from "node:fs";
@@ -58,10 +58,6 @@ export class LettaClient {
...options,
headers: {
"Content-Type": "application/json",
// The server's --secure mode checks X-BARE-PASSWORD ("password <it>") and answers a Bearer
// token alone with 401 (letta/server/rest_api/app.py, 0.6.x). Both are sent: Bearer is what
// later servers read.
"X-BARE-PASSWORD": `password ${this.password}`,
Authorization: `Bearer ${this.password}`,
...(options.headers as Record<string, string> | undefined),
},
+31 -27
View File
@@ -5,29 +5,22 @@
"container-runtime"
],
"requires": [
"postgres-database",
"route"
"postgres-database"
],
"contributes": {
"postgres-database": {
"name": "letta"
},
"route": {
"label": "letta",
"endpoint": "web"
}
},
"binds": {
"postgres-database": "${dir:state}/database.json",
"route": "${dir:state}/route.json"
"postgres-database": "/var/lib/letta/database.json"
},
"secrets": {
"postgres-database": "${dir:state}/database.secret"
"postgres-database": "/var/lib/letta/database.secret"
},
"own-secrets": {
"server-password": "${dir:state}/server-password.secret",
"openai-api-key": "${dir:state}/openai-api-key.secret",
"broker": "${dir:mesh-state}/broker"
"server-password": "/var/lib/letta/server-password.secret",
"broker": "/var/lib/mesh/letta/broker"
},
"listens": [
{
@@ -35,28 +28,28 @@
"port": 8283,
"protocol": "tcp",
"from": "mesh",
"why": "the Letta agent server REST API and web UI, password-protected (--secure); a public name is the route's"
"why": "the Letta agent server REST API and web UI; a public name is a route grant later"
}
],
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/letta",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/letta",
"mode": "0700"
},
{
"id": "server-env",
"type": "file",
"path": "${dir:state}/server.env",
"path": "/var/lib/letta/server.env",
"mode": "0600",
"content": "LETTA_PG_URI=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\nLETTA_SERVER_PASSWORD=${secret:server-password}\nOPENAI_API_KEY=${secret:openai-api-key}\nSECURE=true\nTZ=Europe/Brussels\n"
"content": "LETTA_PG_URI=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\nLETTA_SERVER_PASSWORD=${secret:server-password}\nSECURE=true\nTZ=Europe/Brussels\n"
},
{
"id": "net",
@@ -67,42 +60,53 @@
"id": "server",
"type": "container",
"name": "letta",
"image": "letta/letta@sha256:bfd1e49ce45b9a208c941e832c1d1d194017ff210a3784b0ca6c323aed767a29",
"image": "letta/letta@sha256:1d2e0692514287c5ed1a483e14e16ed945f8632d315539f5e66373bb7d7c471b",
"network": "letta",
"env-file": [
"${dir:state}/server.env"
"/var/lib/letta/server.env"
],
"ports": [
"8283"
],
"secrets-in-environment": "letta 0.6.x reads its settings from the environment only (pydantic settings, no secrets_dir or _FILE twin), and its startup.sh starts an embedded PostgreSQL unless LETTA_PG_URI is set - so the database password travels inside that URI (startup.sh also echoes it to the log); LETTA_SERVER_PASSWORD and OPENAI_API_KEY have no file source either"
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted"
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/letta/config.json",
"mode": "0600",
"content": "{\n \"password\": \"${secret:server-password}\"\n}\n",
"content": "{}\n",
"merge": "json"
},
{
"id": "runtime-env",
"type": "file",
"path": "/var/lib/letta/runtime.env",
"mode": "0600",
"content": "MESH_LETTA_PASSWORD=${secret:server-password}\n"
},
{
"id": "runtime",
"type": "container",
"name": "mesh-letta",
"network": "letta",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro"
"/var/lib/mesh/letta/broker:/run/secrets/broker:ro",
"/var/lib/mesh/letta/config.json:/run/config/config.json:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_LETTA_URL": "http://letta:8283",
"MESH_LETTA_CONFIG_FILE": "/run/config/config.json"
},
"env-file": [
"/var/lib/letta/runtime.env"
],
"restart-on": [
"runtime-config"
],
"artifact": "runtime"
"artifact": "runtime",
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted"
}
],
"build": {
+7 -9
View File
@@ -10,7 +10,7 @@
],
"consumes": [],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/lidarr/broker"
},
"listens": [
{
@@ -23,12 +23,10 @@
],
"accesses": [
{
"id": "music",
"path": "/services/media/music",
"mode": "read-write"
},
{
"id": "downloads",
"path": "/services/media/downloads",
"mode": "read-write"
}
@@ -37,8 +35,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/lidarr",
"mode": "0700"
},
{
"id": "config",
@@ -62,8 +60,8 @@
],
"volumes": [
"/services/lidarr/config:/config",
"${access:music}:/music",
"${access:downloads}:/downloads"
"/services/media/music:/music",
"/services/media/downloads:/downloads"
]
},
{
@@ -72,7 +70,7 @@
"name": "mesh-lidarr",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"/var/lib/mesh/lidarr/broker:/run/secrets/broker:ro",
"/services/lidarr/config:/var/lib/lidarr/config:ro"
],
"env": {
@@ -93,7 +91,7 @@
}
},
"binds": {
"route": "${dir:mesh-state}/route.json"
"route": "/var/lib/mesh/lidarr/route.json"
},
"build": {
"on": [
+5 -4
View File
@@ -6,24 +6,25 @@
"model-access"
],
"binds": {
"model-access": "${dir:state}/model.json"
"model-access": "/var/lib/local-model-consumer/model.json"
},
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/local-model-consumer",
"mode": "0700"
},
{
"id": "config",
"type": "directory",
"path": "/var/lib/local-model-consumer/config",
"mode": "0700"
},
{
"id": "openai-env",
"type": "file",
"path": "${dir:config}/openai.env",
"path": "/var/lib/local-model-consumer/config/openai.env",
"mode": "0600",
"content": "OPENAI_BASE_URL=http://${bound:model-access:at}:${bound:model-access:port}/v1\nOPENAI_MODEL=${bound:model-access:model}\nOPENAI_API_KEY=local\n"
}
-17
View File
@@ -1,17 +0,0 @@
# mailu
Mail — Mailu, with its provisioner (the `smtp` provision) and tools, on the tool runtime.
## Settings
A definition names no mesh (novox/hq ADR 0112, ADR 0155), so the values that are this
installation's are settings on the assignment, `settings set mailu <file>`:
```json
{"domain": "…", "sitename": "…", "website": "https://…", "proxy-address": "…"}
```
`domain` is the mail domain (also the provisioner's, for a consumer's address); `sitename` and
`website` are shown by the web front; `proxy-address` is what `REAL_IP_FROM` trusts a real-IP
header from — the address the proxy forwards with. The front's own hostname is the name of its
`web` route, told to it by the mesh.
+21 -16
View File
@@ -117,35 +117,35 @@
},
{
"name": "web",
"port": 7080,
"port": 80,
"protocol": "tcp",
"from": "mesh",
"why": "the web front over http; only the ACME HTTP-01 passthrough is routed here — everything else 301s to https and would loop a proxy"
"why": "the web front over http; only the ACME HTTP-01 passthrough is routed here \u2014 everything else 301s to https and would loop a proxy"
},
{
"name": "web-tls",
"port": 7443,
"port": 443,
"protocol": "tcp",
"from": "mesh",
"why": "the web front over its own TLS (admin, webmail, API); its public name is a route grant reaching it here"
"why": "the web front over its own TLS (admin, webmail, API); the public name mail.novox.be is a route grant reaching it here"
},
{
"name": "autoconfig",
"port": 4243,
"protocol": "tcp",
"from": "mesh",
"why": "automx: mail client autoconfiguration; the autoconfig, autodiscover and automx names are route grants reaching it here"
"why": "automx: mail client autoconfiguration; autoconfig/autodiscover/automx.novox.be are route grants reaching it here"
}
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/mailu/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/mailu",
"mode": "0700"
},
{
"id": "state",
@@ -168,7 +168,7 @@
"type": "file",
"path": "${dir:state}/mailu.env",
"mode": "0644",
"content": "ADMIN_ADDRESS=mailu-admin\nANTISPAM_ADDRESS=mailu-antispam\nANTIVIRUS_ADDRESS=mailu-antivirus\nIMAP_ADDRESS=mailu-imap\nSMTP_ADDRESS=mailu-smtp\nFRONT_ADDRESS=mailu-front\nWEBMAIL_ADDRESS=mailu-webmail\nWEBDAV_ADDRESS=mailu-webdav\nREDIS_ADDRESS=mailu-redis\nPORTS=25,80,443,465,993,995,4190,110,143,587\nDOMAIN=${setting:domain}\nHOSTNAMES=${bound:route:name-web}\nPOSTMASTER=admin\nSITENAME=${setting:sitename}\nWEBSITE=${setting:website}\nTLS_FLAVOR=letsencrypt\nSUBNET=192.168.203.0/24\nCOMPOSE_PROJECT_NAME=mailu\nANTIVIRUS=clamav\nWEBMAIL=roundcube\nWEBDAV=radicale\nFETCHMAIL_ENABLED=True\nFETCHMAIL_DELAY=600\nADMIN=true\nWEB_ADMIN=/admin\nWEB_WEBMAIL=/webmail\nWEBROOT_REDIRECT=/webmail\nAPI=true\nWEB_API=/api\nAUTH_RATELIMIT_IP=6000/hour\nAUTH_RATELIMIT_USER=1000/day\nCREDENTIAL_ROUNDS=12\nPASSWORD_SCHEME=PBKDF2\nDISABLE_STATISTICS=True\nMESSAGE_SIZE_LIMIT=50000000\nMESSAGE_RATELIMIT=200/day\nRECIPIENT_DELIMITER=+\nPOSTFIX_MYNETWORKS=127.0.0.0/8 [::1]/128\nRELAYNETS=\nRELAYHOST=\nREJECT_UNLISTED_RECIPIENT=\nDB_FLAVOR=postgresql\nINITIAL_ADMIN_ACCOUNT=admin\nINITIAL_ADMIN_DOMAIN=${setting:domain}\nINITIAL_ADMIN_MODE=ifmissing\nSMTP_PORT=25\nSMTPS_PORT=465\nSUBMISSION_PORT=587\nPOP3_PORT=110\nPOP3S_PORT=995\nIMAP_PORT=143\nIMAPS_PORT=993\nHTTP_PORT=7080\nHTTPS_PORT=7443\nAUTOMX_PORT=4243\nAMX_SMTP_ADDRESS=${bound:route:name-web}\nAMX_SMTP_PORT=587\nAMX_IMAP_ADDRESS=${bound:route:name-web}\nAMX_IMAP_PORT=143\nAMX_MAIL_DOMAINS=${setting:domain}\nDMARC_RUA=admin\nDMARC_RUF=admin\nLETSENCRYPT_SHORTCHAIN=True\nTZ=Etc/UTC\nLOG_LEVEL=INFO\nWELCOME=false\nREAL_IP_HEADER=X-Real-IP\nREAL_IP_FROM=${setting:proxy-address}\nCOMPRESSION=\nCOMPRESS_LEVEL=\nCOMPRESSION_LEVEL=\nBIND_ADDRESS4=127.0.0.1\nBIND_ADDRESS6=::1\nMAILU_VERSION=1.9\nDOCKER_ORG=mailu\nDOCKER_PREFIX=\nWELCOME_SUBJECT=Welcome to your new email account\nWELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly!\n"
"content": "ADMIN_ADDRESS=mailu-admin\nANTISPAM_ADDRESS=mailu-antispam\nANTIVIRUS_ADDRESS=mailu-antivirus\nIMAP_ADDRESS=mailu-imap\nSMTP_ADDRESS=mailu-smtp\nFRONT_ADDRESS=mailu-front\nWEBMAIL_ADDRESS=mailu-webmail\nWEBDAV_ADDRESS=mailu-webdav\nREDIS_ADDRESS=mailu-redis\nPORTS=25,80,443,465,993,995,4190,110,143,587\nDOMAIN=novox.be\nHOSTNAMES=mail.novox.be\nPOSTMASTER=admin\nSITENAME=Novox\nWEBSITE=https://novox.be\nTLS_FLAVOR=letsencrypt\nSUBNET=192.168.203.0/24\nCOMPOSE_PROJECT_NAME=mailu\nANTIVIRUS=clamav\nWEBMAIL=roundcube\nWEBDAV=radicale\nFETCHMAIL_ENABLED=True\nFETCHMAIL_DELAY=600\nADMIN=true\nWEB_ADMIN=/admin\nWEB_WEBMAIL=/webmail\nWEBROOT_REDIRECT=/webmail\nAPI=true\nWEB_API=/api\nAUTH_RATELIMIT_IP=6000/hour\nAUTH_RATELIMIT_USER=1000/day\nCREDENTIAL_ROUNDS=12\nPASSWORD_SCHEME=PBKDF2\nDISABLE_STATISTICS=True\nMESSAGE_SIZE_LIMIT=50000000\nMESSAGE_RATELIMIT=200/day\nRECIPIENT_DELIMITER=+\nPOSTFIX_MYNETWORKS=127.0.0.0/8 [::1]/128\nRELAYNETS=\nRELAYHOST=\nREJECT_UNLISTED_RECIPIENT=\nDB_FLAVOR=postgresql\nINITIAL_ADMIN_ACCOUNT=admin\nINITIAL_ADMIN_DOMAIN=novox.be\nINITIAL_ADMIN_MODE=ifmissing\nSMTP_PORT=25\nSMTPS_PORT=465\nSUBMISSION_PORT=587\nPOP3_PORT=110\nPOP3S_PORT=995\nIMAP_PORT=143\nIMAPS_PORT=993\nHTTP_PORT=${port:80}\nHTTPS_PORT=${port:443}\nAUTOMX_PORT=4243\nAMX_SMTP_ADDRESS=mail.novox.be\nAMX_SMTP_PORT=587\nAMX_IMAP_ADDRESS=mail.novox.be\nAMX_IMAP_PORT=143\nAMX_MAIL_DOMAINS=novox.be\nDMARC_RUA=admin\nDMARC_RUF=admin\nLETSENCRYPT_SHORTCHAIN=True\nTZ=Etc/UTC\nLOG_LEVEL=INFO\nWELCOME=false\nREAL_IP_HEADER=X-Real-IP\nREAL_IP_FROM=142.132.152.141\nCOMPRESSION=\nCOMPRESS_LEVEL=\nCOMPRESSION_LEVEL=\nBIND_ADDRESS4=127.0.0.1\nBIND_ADDRESS6=::1\nMAILU_VERSION=1.9\nDOCKER_ORG=mailu\nDOCKER_PREFIX=\nWELCOME_SUBJECT=Welcome to your new email account\nWELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly!\n"
},
{
"id": "secret-env",
@@ -315,7 +315,10 @@
"${dir:data-data}:/data",
"${dir:data-dkim}:/dkim"
],
"secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified"
"secrets-in-environment": "mailu-admin honours SECRET_KEY_FILE, DB_PW_FILE and API_TOKEN_FILE (configuration.py) but INITIAL_ADMIN_PW is env-only (start.py); the remaining containers' need for SECRET_KEY is unverified",
"dns": [
"192.168.203.254"
]
},
{
"id": "imap",
@@ -453,8 +456,8 @@
"587",
"993",
"995",
"7080:80",
"7443:443"
"80",
"443"
],
"volumes": [
"${dir:data-certs}:/certs",
@@ -467,7 +470,7 @@
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/mailu/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -478,10 +481,10 @@
"name": "mesh-mailu",
"network": "mailu",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"/var/lib/mesh/mailu/broker:/run/secrets/broker:ro",
"${dir:state}/api-token.secret:/run/secrets/api-token:ro",
"${dir:grants}:${dir:grants}:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"/var/lib/mesh/mailu/config.json:/run/config/config.json:ro",
"/var/run/docker.sock:/var/run/docker.sock"
],
"env": {
@@ -490,6 +493,7 @@
"MESH_MAILU_API_KEY_FILE": "/run/secrets/api-token",
"MESH_MAILU_IMAP_CONTAINER": "mailu-imap",
"MESH_MAILU_CONFIG_FILE": "/run/config/config.json",
"MESH_MAILU_DOMAIN": "novox.be",
"MESH_RECEIVES": "${dir:grants}/mesh.json"
},
"restart-on": [
@@ -553,7 +557,8 @@
"serves": {
"smtp": {
"port": 587,
"domain": "${setting:domain}"
"domain": "novox.be",
"name": "mail.novox.be"
}
},
"receives": {
+3 -18
View File
@@ -13,32 +13,17 @@
// it to both ends; mailu sets exactly that password every run — so a rotation takes — and seals
// nothing: the consumer already has its copy through the mesh's own channel.
import { readFileSync } from "node:fs";
import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner";
import { MailuClient } from "../client.js";
const mailu = MailuClient.fromEnv();
// The mail server's own domain: the operator's value, from the settings the mesh merges into this
// module's config file (`settings set mailu` with {"domain": …}; novox/hq ADR 0112, ADR 0155). A
// definition names no mesh, so it is never a literal in the manifest — and it used to be, as
// MESH_MAILU_DOMAIN, which is still read for a mesh that has not re-registered the manifest.
// The mail server's own domain. From the environment the manifest composes, because the client's
// config file carries the admin API's coordinates, not the mail domain.
function domain(): string {
const file = process.env.MESH_MAILU_CONFIG_FILE;
if (file) {
try {
const config = JSON.parse(readFileSync(file, "utf8")) as { domain?: unknown };
if (typeof config.domain === "string" && config.domain.trim() !== "") return config.domain.trim();
} catch {
// Unreadable or not JSON: fall through to the environment, and the error below names both.
}
}
const named = (process.env.MESH_MAILU_DOMAIN ?? "").trim();
if (named === "") {
throw new Error(
"no mail domain is set, so a consumer's address cannot be composed — `settings set mailu <file>` " +
'with {"domain": "<the mail domain>"}',
);
throw new Error("MESH_MAILU_DOMAIN is not set, so a consumer's address cannot be composed");
}
return named;
}
+2 -2
View File
@@ -17,8 +17,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/marrytts",
"mode": "0700"
},
{
"id": "server",
-125
View File
@@ -1,125 +0,0 @@
{
"module": "matrix",
"version": "1",
"capabilities": [
"container-runtime"
],
"listens": [
{
"name": "client",
"port": 6167,
"protocol": "tcp",
"from": "mesh",
"why": "Conduit's client-server and federation APIs over plain HTTP. Both arrive through the route on 443: Conduit answers /.well-known/matrix/server with <its name>:443, so other homeservers federate through the proxy and nothing needs the traditional 8448"
},
{
"name": "web",
"port": 80,
"protocol": "tcp",
"from": "mesh",
"why": "Element Web, the static browser client, served by the image's nginx; reached through its route"
}
],
"requires": [
"route"
],
"contributes": {
"route": {
"homeserver": {
"label": "matrix",
"endpoint": "client"
},
"element": {
"label": "element",
"endpoint": "web"
}
}
},
"binds": {
"route": "${dir:state}/route.json"
},
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
},
{
"id": "db",
"type": "directory",
"mode": "0700"
},
{
"id": "conduit-conf",
"type": "file",
"path": "${dir:state}/conduit.toml",
"mode": "0644",
"content": "# Written by the mesh (modules/matrix). Conduit reads this file (CONDUIT_CONFIG); nothing comes\n# from the environment. server_name is the homeserver's permanent identity: every user id, room id\n# and signature in the database carries it, so it is the name this module is served under\n# (${bound:route:name-homeserver}) and never changes once a database exists.\n[global]\nserver_name = \"${bound:route:name-homeserver}\"\ndatabase_backend = \"rocksdb\"\ndatabase_path = \"/var/lib/matrix-conduit/\"\naddress = \"0.0.0.0\"\nport = 6167\nmax_request_size = 20000000\nallow_registration = false\nallow_federation = true\nallow_check_for_updates = true\ntrusted_servers = [\"matrix.org\"]\n",
"names-on-purpose": {
"matrix.org": "the federation's public key server, trusted by default; the world's, not this mesh's"
}
},
{
"id": "element-conf",
"type": "file",
"path": "${dir:state}/element.json",
"mode": "0644",
"merge": "json",
"content": "{\n \"default_server_name\": \"${bound:route:name-homeserver}\",\n \"default_server_config\": {\n \"m.homeserver\": {\n \"base_url\": \"https://${bound:route:name-homeserver}\"\n },\n \"m.identity_server\": {\n \"base_url\": \"https://vector.im\"\n }\n },\n \"brand\": \"Element\",\n \"integrations_ui_url\": \"https://scalar.vector.im/\",\n \"integrations_rest_url\": \"https://scalar.vector.im/api\",\n \"integrations_widgets_urls\": [\n \"https://scalar.vector.im/_matrix/integrations/v1\",\n \"https://scalar.vector.im/api\",\n \"https://scalar-staging.vector.im/_matrix/integrations/v1\",\n \"https://scalar-staging.vector.im/api\",\n \"https://scalar-staging.riot.im/scalar/api\"\n ],\n \"bug_report_endpoint_url\": \"https://element.io/bugreports/submit\",\n \"uisi_autorageshake_app\": \"element-auto-uisi\",\n \"show_labs_settings\": true,\n \"room_directory\": {\n \"servers\": [\n \"${bound:route:name-homeserver}\",\n \"matrix.org\",\n \"gitter.im\",\n \"libera.chat\"\n ]\n },\n \"enable_presence_by_hs_url\": {\n \"https://matrix.org\": false,\n \"https://matrix-client.matrix.org\": false\n },\n \"terms_and_conditions_links\": [\n {\n \"url\": \"https://element.io/privacy\",\n \"text\": \"Privacy Policy\"\n },\n {\n \"url\": \"https://element.io/cookie-policy\",\n \"text\": \"Cookie Policy\"\n }\n ],\n \"features\": {\n \"feature_video_rooms\": true,\n \"feature_rust_crypto\": true\n },\n \"element_call\": {\n \"url\": \"https://call.element.dev\"\n }\n}\n",
"names-on-purpose": {
"matrix.org": "the public room directory and the federation's largest homeserver; the world's",
"matrix-client.matrix.org": "the same homeserver's client endpoint; the world's",
"vector.im": "Element's public identity server; the world's",
"scalar.vector.im": "Element's public integration manager; the world's",
"scalar-staging.vector.im": "Element's staging integration manager, named by the upstream default config; the world's",
"scalar-staging.riot.im": "the same, under its former name; the world's",
"element.io": "Element's bug reports, privacy and cookie pages; the world's",
"gitter.im": "a public room directory; the world's",
"libera.chat": "a public room directory; the world's",
"call.element.dev": "Element Call's public instance; the world's"
}
},
{
"id": "net",
"type": "network",
"name": "matrix"
},
{
"id": "homeserver",
"type": "container",
"name": "matrix",
"image": "matrixconduit/matrix-conduit@sha256:b0d24248e94f944ca49f90f10c429e3d65f4472bdde25661ecea9840134fb133",
"network": "matrix",
"env": {
"CONDUIT_CONFIG": "/etc/conduit/conduit.toml"
},
"ports": [
"6167"
],
"volumes": [
"${dir:db}:/var/lib/matrix-conduit",
"${dir:state}/conduit.toml:/etc/conduit/conduit.toml:ro"
],
"restart-on": [
"conduit-conf"
]
},
{
"id": "element",
"type": "container",
"name": "element-web",
"image": "vectorim/element-web@sha256:a8f415462ab8d2600a592ba1b92bea51efe5a4d10eb738aab9bed769f7099613",
"network": "matrix",
"ports": [
"80"
],
"volumes": [
"${dir:state}/element.json:/app/config.json:ro"
],
"restart-on": [
"element-conf"
]
}
]
}
+11 -11
View File
@@ -20,13 +20,13 @@
}
},
"binds": {
"postgres-database": "${dir:state}/database.json"
"postgres-database": "/var/lib/mesh-catalog/database.json"
},
"secrets": {
"postgres-database": "${dir:state}/database.secret"
"postgres-database": "/var/lib/mesh-catalog/database.secret"
},
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/mesh-catalog/broker"
},
"consumes": [
"mesh-build-machine.built",
@@ -43,19 +43,19 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/mesh-catalog",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/mesh-catalog",
"mode": "0700"
},
{
"id": "database-url",
"type": "file",
"path": "${dir:state}/database.url",
"path": "/var/lib/mesh-catalog/database.url",
"mode": "0600",
"content": "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n"
},
@@ -65,9 +65,9 @@
"name": "mesh-catalog",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:state}:/run/state",
"${dir:state}/database.url:/run/secrets/database-url:ro"
"/var/lib/mesh/mesh-catalog/broker:/run/secrets/broker:ro",
"/var/lib/mesh-catalog:/run/state",
"/var/lib/mesh-catalog/database.url:/run/secrets/database-url:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
-13
View File
@@ -1,13 +0,0 @@
# The console (novox/hq ADR 0152, design 34): the mesh's tools for whoever is on a machine, served
# over MCP on that machine's loopback.
#
# **Nothing is compiled here.** The console is the tool runtime's own client — `mesh serve` — which
# the runtime image already carries beside the runtime it runs modules with. This recipe changes the
# program the image starts and nothing else, so the console is exactly the client a person can run by
# hand, started by the mesh instead, on the credential the mesh sealed to the machine.
#
# One base, named rather than pinned: the mesh answers with the copy it holds (novox/hq issue 044).
ARG RUNTIME_BASE
FROM ${RUNTIME_BASE}
ENTRYPOINT ["node", "dist/mesh.js"]
-38
View File
@@ -1,38 +0,0 @@
# mesh-console
The mesh's tools, on the machine a person sits at, served by a module the mesh assigned there
(novox/hq [ADR 0152](https://git.novox.be/novox/hq), design 34).
Assign it to a machine and an agent on that machine has the mesh's tools at
`http://127.0.0.1:<port>/mcp` — MCP over HTTP, `initialize`, `tools/list`, `tools/call`. A person at
a terminal reaches the same endpoint with `mesh tools --console http://127.0.0.1:<port>` and
`mesh call <module>.<tool> --console …`, with no credential of their own: the console holds it.
## What it is
The tool runtime's own client, `mesh serve`, started by the mesh on the credential it sealed to the
machine for `<node>.mesh-console`. The manifest says three things nothing else in the catalogue says
together:
- `invokes: ["*"]` — it calls every tool on the mesh, and the bus grants exactly that publish side;
- a listener `from: machine` — loopback only, and the filter opens nothing for it;
- no `emits`, no `consumes`, no `tools` — nothing on the bus can address it.
**Loopback is the authority boundary.** Whoever can connect is on the machine, and whoever is on the
machine is the account that owns the mesh there (ADR 0034, ADR 0144). There is no token and no login,
and `mesh serve` refuses to bind anything but a loopback address.
## What it lists
What the running modules answer: every tool runtime serves a `tools` verb for its module, and the
console asks the catalogue which modules the mesh holds and each module what it serves. A module that
did not answer — not assigned, not up, or built before the runtime answered `tools` — is named in the
list's `_meta.notAnswering` and can still be called by `<module>.<tool>`.
The mesh's own verbs (`status`, `push`, `assign`) are the `mesh-controller` seat's tools under
ADR 0132 and are not served on the bus yet; they appear here when they are.
## Port
The manifest declares port 4270 and the mesh assigns the machine port as it does for any listener;
the console binds `127.0.0.1:${port:4270}`. `node show <machine>` says which port a machine was given.
-64
View File
@@ -1,64 +0,0 @@
{
"module": "mesh-console",
"version": "1",
"slug": "console",
"capabilities": [
"container-runtime"
],
"invokes": [
"*"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
},
"listens": [
{
"name": "mcp",
"port": 4270,
"protocol": "tcp",
"from": "machine",
"why": "the mesh's tools for whoever is on this machine, over MCP on loopback; the machine's login is the authority (novox/hq ADR 0152)"
}
],
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
},
{
"id": "server",
"type": "container",
"name": "mesh-console",
"network": "host",
"args": [
"serve"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_CONSOLE_LISTEN": "127.0.0.1:${port:4270}"
},
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro"
],
"artifact": "runtime"
}
],
"build": {
"on": [
{
"arg": "RUNTIME_BASE",
"module": "mesh-tools",
"artifact": "runtime"
}
],
"artifacts": [
{
"name": "runtime",
"kind": "image",
"from": "Dockerfile"
}
]
}
}
+17 -14
View File
@@ -21,41 +21,44 @@
"mesh-vault.secret.deprovisioned"
],
"receives": {
"secret": "${dir:grants}/mesh.json"
"secret": "/var/lib/mesh-vault/grants/mesh.json"
},
"grants": {
"secret": "${dir:grants}"
"secret": "/var/lib/mesh-vault/grants"
},
"keeps": "/var/lib/mesh-vault/root",
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/mesh-vault/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/mesh-vault",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/mesh-vault",
"mode": "0700"
},
{
"id": "grants",
"type": "directory",
"path": "/var/lib/mesh-vault/grants",
"mode": "0700"
},
{
"id": "ledger",
"type": "directory",
"path": "/var/lib/mesh-vault/ledger",
"mode": "0700"
},
{
"id": "root",
"type": "directory",
"path": "/var/lib/mesh-vault/root",
"mode": "0700"
},
{
@@ -64,16 +67,16 @@
"name": "mesh-vault",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:grants}:${dir:grants}:ro",
"${dir:ledger}:${dir:ledger}",
"${dir:root}:${dir:root}:ro"
"/var/lib/mesh/mesh-vault/broker:/run/secrets/broker:ro",
"/var/lib/mesh-vault/grants:/var/lib/mesh-vault/grants:ro",
"/var/lib/mesh-vault/ledger:/var/lib/mesh-vault/ledger",
"/var/lib/mesh-vault/root:/var/lib/mesh-vault/root:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_RECEIVES": "${dir:grants}/mesh.json",
"MESH_VAULT_LEDGER": "${dir:ledger}",
"MESH_VAULT_ROOT": "${dir:root}"
"MESH_RECEIVES": "/var/lib/mesh-vault/grants/mesh.json",
"MESH_VAULT_LEDGER": "/var/lib/mesh-vault/ledger",
"MESH_VAULT_ROOT": "/var/lib/mesh-vault/root"
},
"artifact": "runtime"
}
+18 -16
View File
@@ -53,39 +53,40 @@
}
},
"receives": {
"s3-bucket": "${dir:grants}/mesh.json"
"s3-bucket": "/var/lib/minio/grants/mesh.json"
},
"grants": {
"s3-bucket": "${dir:grants}"
"s3-bucket": "/var/lib/minio/grants"
},
"own-secrets": {
"root": "${dir:state}/root.secret",
"broker": "${dir:mesh-state}/broker"
"root": "/var/lib/minio/root.secret",
"broker": "/var/lib/mesh/minio/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/minio",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/minio",
"mode": "0700"
},
{
"id": "grants",
"type": "directory",
"path": "/var/lib/minio/grants",
"mode": "0700"
},
{
"id": "root-env",
"type": "file",
"path": "${dir:state}/root.env",
"path": "/var/lib/minio/root.env",
"mode": "0600",
"content": "MINIO_ROOT_USER=meshroot\nMINIO_BROWSER_REDIRECT_URL=https://${bound:route:name-console}\n"
"content": "MINIO_ROOT_USER=meshroot\n"
},
{
"id": "data",
@@ -111,7 +112,7 @@
":9001"
],
"env-file": [
"${dir:state}/root.env"
"/var/lib/minio/root.env"
],
"ports": [
"9000",
@@ -119,10 +120,11 @@
],
"volumes": [
"/var/lib/minio-store:/data",
"${dir:state}/root.secret:/run/secrets/root:ro"
"/var/lib/minio/root.secret:/run/secrets/root:ro"
],
"env": {
"MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root",
"MINIO_BROWSER_REDIRECT_URL": "https://files.novox.be",
"MINIO_REGION": "eu-west"
}
},
@@ -132,9 +134,9 @@
"name": "mesh-minio",
"network": "minio-net",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:grants}:${dir:grants}:ro",
"${dir:state}/root.secret:/run/secrets/root:ro"
"/var/lib/mesh/minio/broker:/run/secrets/broker:ro",
"/var/lib/minio/grants:/var/lib/minio/grants:ro",
"/var/lib/minio/root.secret:/run/secrets/root:ro"
],
"env": {
"MESH_MINIO_ENDPOINT": "http://minio:9000",
@@ -142,7 +144,7 @@
"MESH_MINIO_ROOT_PASSWORD_FILE": "/run/secrets/root",
"MESH_MINIO_REGION": "eu-west",
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_RECEIVES": "${dir:grants}/mesh.json"
"MESH_RECEIVES": "/var/lib/minio/grants/mesh.json"
},
"artifact": "runtime"
}
+11 -11
View File
@@ -14,34 +14,34 @@
}
},
"binds": {
"postgres-database": "${dir:state}/database.json"
"postgres-database": "/var/lib/model-usage/database.json"
},
"secrets": {
"postgres-database": "${dir:state}/database.secret"
"postgres-database": "/var/lib/model-usage/database.secret"
},
"consumes": [
"*.usage.*"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/model-usage/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/model-usage",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/model-usage",
"mode": "0700"
},
{
"id": "database-url",
"type": "file",
"path": "${dir:state}/database.url",
"path": "/var/lib/model-usage/database.url",
"mode": "0600",
"content": "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n"
},
@@ -52,9 +52,9 @@
"image": "mesh-runtime-model-usage@sha256:0000000000000000000000000000000000000000000000000000000000000000",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:state}:/run/state",
"${dir:state}/database.url:/run/secrets/database-url:ro"
"/var/lib/mesh/model-usage/broker:/run/secrets/broker:ro",
"/var/lib/model-usage:/run/state",
"/var/lib/model-usage/database.url:/run/secrets/database-url:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
+4 -4
View File
@@ -40,15 +40,15 @@
},
"own-secrets": {
"root": "${dir:state}/root.secret",
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/mongodb/broker"
},
"secrets-owner": "999:999",
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/mongodb",
"mode": "0700"
},
{
"id": "state",
@@ -95,7 +95,7 @@
"name": "mesh-mongodb",
"network": "mongodb",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"/var/lib/mesh/mongodb/broker:/run/secrets/broker:ro",
"${dir:grants}:${dir:grants}:ro",
"${dir:state}/root.secret:/run/secrets/root:ro"
],
+1 -1
View File
@@ -13,7 +13,7 @@ ARG RUNTIME_BASE
FROM ${BUILD_BASE} AS build
WORKDIR /app/modules/mosquitto
COPY . .
RUN node /app/node_modules/typescript/bin/tsc topics.ts client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
RUN node /app/node_modules/typescript/bin/tsc client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
FROM ${RUNTIME_BASE}
+24 -38
View File
@@ -20,8 +20,6 @@ import { readFileSync } from "node:fs";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { missingAcls, parseRoleAcls, staleAcls, wantedAcls } from "./topics.js";
const run = promisify(execFile);
export interface MqttConn {
@@ -143,19 +141,14 @@ export class MosquittoClient {
}
/**
* Create (or reset to a known state) a client granted exactly these topic filters, idempotently.
* The grant is a same-named role carrying, for every filter, publish, receive and subscribe — and
* nothing else: an ACL the role carries that the filters no longer name is removed, so narrowing a
* consumer's `topics` narrows what it may do. By default the filters are the consumer's own
* subtree, `<as>/#` (see topics.ts). Called again for an existing client, it resets the password
* and re-asserts the ACLs.
*
* Only the role named for this client is ever changed. A client or role the mesh did not make —
* a device carried from the predecessor's password file, its `legacy-full-access` role — is never
* read, changed or removed here.
* Create (or reset to a known state) a client scoped to one topic namespace, idempotently. The
* client is confined to `<prefix>/#` by a same-named role: it may publish to, subscribe to and
* receive on exactly its own subtree and nothing else — the MQTT analog of redis's keyspace-scoped
* ACL user. Called again for an existing client, it resets the password and re-asserts the ACLs.
*/
async createScopedClient(username: string, password: string, filters: readonly string[]): Promise<void> {
async createScopedClient(username: string, password: string, topicPrefix: string): Promise<void> {
const role = username; // one role per client, named for it
const pattern = `${topicPrefix}/#`;
if (await this.clientExists(username)) {
await this.ctl("setClientPassword", username, password);
@@ -168,18 +161,17 @@ export class MosquittoClient {
await this.ctl("createClient", username, "-p", password);
}
// createRole and addRoleACL are one-shot: each rejects with an "already exists" when re-run
// against a role/ACL it created on a previous reconcile. That rejection is the intended terminal
// state, so it is swallowed.
// A role carrying exactly this client's topic ACLs. createRole, addRoleACL and addClientRole are
// all one-shot: each rejects with an "already exists" when re-run against a role/ACL/binding it
// created on a previous reconcile. That rejection is the intended terminal state — the ACL is
// deterministic (`<prefix>/#`, allow), so re-adding the identical entry is a no-op — so it is
// swallowed. (Until the exit code was fixed this was invisible: the tool returned 0 and the
// rejection was lost; now it surfaces, and each of these adds must tolerate its own idempotent
// re-run explicitly.)
await ignoreExisting(this.ctl("createRole", role));
const wanted = wantedAcls(filters);
const current = parseRoleAcls(await this.ctl("getRole", role));
for (const acl of missingAcls(current, wanted)) {
await ignoreExisting(this.ctl("addRoleACL", role, acl.type, acl.topic, "allow"));
}
// What the consumer no longer asks for — added before it narrowed its topics — is taken away.
for (const acl of staleAcls(current, wanted)) {
await ignoreMissing(this.ctl("removeRoleACL", role, acl.type, acl.topic));
for (const acl of ["publishClientSend", "publishClientReceive", "subscribePattern"]) {
// allow (1) this client to send to, receive on, and subscribe under its own subtree.
await ignoreExisting(this.ctl("addRoleACL", role, acl, pattern, "allow"));
}
// Bind the role only when it is not already bound — addClientRole is the one call whose
// idempotent re-run cannot be recognised by message (see clientHasRole).
@@ -189,31 +181,25 @@ export class MosquittoClient {
}
/**
* Whether a consumer's client accepts exactly this password, still carries its own role, and that
* role grants exactly these filters. Read-only. The password is checked the way the consumer is
* checked, by an MQTT CONNECT as it, and the broker's CONNACK code is the answer: 0 accepted,
* 4 bad credentials, 5 not authorised. Nothing rides on argv. An unreachable broker rejects
* (novox/hq issue 120).
* Whether a consumer's client accepts exactly this password and still carries its own role.
* Read-only. The password is checked the way the consumer is checked, by an MQTT CONNECT as it,
* and the broker's CONNACK code is the answer: 0 accepted, 4 bad credentials, 5 not authorised.
* Nothing rides on argv. An unreachable broker rejects (novox/hq issue 120).
*/
async holdsClient(username: string, password: string, filters: readonly string[]): Promise<boolean> {
async holdsClient(username: string, password: string): Promise<boolean> {
const code = await mqttConnack(this.conn.host, this.conn.port, username, password);
if (code === 4 || code === 5) return false;
if (code !== 0) throw new Error(`mosquitto refused ${username} with CONNACK ${code}`);
// The role, asked directly: only "not found" means absent. Any other failure to ask rejects,
// unlike clientHasRole, which reads every failure as "no role".
let client: string;
let role: string;
let out: string;
try {
client = await this.ctl("getClient", username);
role = await this.ctl("getRole", username);
out = await this.ctl("getClient", username);
} catch (err) {
if (/not\s*found|does not exist|no such/i.test(String(err))) return false;
throw err;
}
if (!new RegExp(`(^|\\s)${escapeRegExp(username)}\\s+\\(priority`, "m").test(client)) return false;
const current = parseRoleAcls(role);
const wanted = wantedAcls(filters);
return missingAcls(current, wanted).length === 0 && staleAcls(current, wanted).length === 0;
return new RegExp(`(^|\\s)${escapeRegExp(username)}\\s+\\(priority`, "m").test(out);
}
/** Remove a client and the per-client role created for it, idempotently. */
+22 -23
View File
@@ -20,20 +20,17 @@
"mosquitto.topic.deprovisioned"
],
"serves": {
"mqtt-topic": {
"scheme": "mqtt",
"port": 1883
}
"mqtt-topic": {}
},
"receives": {
"mqtt-topic": "${dir:grants}/mesh.json"
"mqtt-topic": "/var/lib/mosquitto-module/grants/mesh.json"
},
"grants": {
"mqtt-topic": "${dir:grants}"
"mqtt-topic": "/var/lib/mosquitto-module/grants"
},
"own-secrets": {
"admin": "${dir:mesh-state}/admin",
"broker": "${dir:mesh-state}/broker"
"admin": "/var/lib/mosquitto-module/admin.secret",
"broker": "/var/lib/mesh/mosquitto/broker"
},
"listens": [
{
@@ -55,30 +52,32 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/mosquitto",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/mosquitto-module",
"mode": "0700"
},
{
"id": "grants",
"id": "grants-dir",
"type": "directory",
"path": "/var/lib/mosquitto-module/grants",
"mode": "0700"
},
{
"id": "data",
"type": "directory",
"path": "/services/mosquitto/data",
"mode": "0700",
"owner": "1883:1883"
},
{
"id": "server-conf",
"type": "file",
"path": "${dir:state}/mosquitto.conf",
"path": "/var/lib/mosquitto-module/mosquitto.conf",
"mode": "0600",
"owner": "1883:1883",
"content": "persistence true\npersistence_location /mosquitto/data\n\nlog_dest stdout\nlog_type warning\nlog_type error\nlog_type notice\n\n# Every client authenticates; identities and their per-topic ACLs are managed\n# at runtime by the dynamic security plugin, whose store the plugin itself owns.\nallow_anonymous false\nplugin /usr/lib/mosquitto_dynamic_security.so\nplugin_opt_config_file /mosquitto/data/dynamic-security.json\n\n# MQTT listener\nlistener 1883\n\n# MQTT-over-WebSockets listener\nlistener 8081\nprotocol websockets\n"
@@ -94,8 +93,8 @@
"name": "mosquitto-bootstrap",
"run-once": true,
"volumes": [
"${dir:data}:/mosquitto/data",
"${dir:mesh-state}/admin:/run/secrets/admin:ro"
"/services/mosquitto/data:/mosquitto/data",
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro"
],
"env": {
"MESH_PROVISION_MQTT": "mosquitto:1883",
@@ -113,15 +112,15 @@
"id": "server",
"type": "container",
"name": "mosquitto",
"image": "eclipse-mosquitto@sha256:38c0da4f2ef84284d47b3b3eeea1cb3bdeabe81ee10caf0cd5c5ff61ee3ea408",
"image": "eclipse-mosquitto@sha256:6f8d8a947c506f8a2290ec65cd4bd2bc7cb4d43fb5f6271f861cb013e2ef9797",
"network": "mosquitto",
"ports": [
"1883",
"8081"
],
"volumes": [
"${dir:data}:/mosquitto/data",
"${dir:state}/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro"
"/services/mosquitto/data:/mosquitto/data",
"/var/lib/mosquitto-module/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro"
]
},
{
@@ -130,13 +129,13 @@
"name": "mesh-mosquitto",
"network": "mosquitto",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:grants}:${dir:grants}:ro",
"${dir:mesh-state}/admin:/run/secrets/admin:ro"
"/var/lib/mesh/mosquitto/broker:/run/secrets/broker:ro",
"/var/lib/mosquitto-module/grants:/var/lib/mosquitto-module/grants:ro",
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_RECEIVES": "${dir:grants}/mesh.json",
"MESH_RECEIVES": "/var/lib/mosquitto-module/grants/mesh.json",
"MESH_PROVISION_MQTT": "mosquitto:1883",
"MESH_PROVISION_ADMIN_USER": "mesh-admin",
"MESH_PROVISION_PASSWORD_FILE": "/run/secrets/admin"
+1 -6
View File
@@ -1,14 +1,9 @@
{
"name": "@novox/module-mosquitto",
"version": "0.1.0",
"description": "mosquitto \u2014 provides the mesh mqtt-topic interface. Its admin client, provisioner, tools and events live here (novox/hq ADR 0039).",
"description": "mosquitto — provides the mesh mqtt-topic interface. Its admin client, provisioner, tools and events live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"scripts": {
"build": "tsc topics.ts client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
"typecheck": "tsc -p tsconfig.json",
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
},
"dependencies": {
"@novox/mesh-sdk": "^0.1.1"
},
+6 -24
View File
@@ -5,14 +5,7 @@
//
// The `mqtt-topic` interface: a consumer connects as `as` with the password the mesh minted, and
// publishes and subscribes under `<as>/#`, isolated from every other consumer by a Dynamic Security
// role scoped to exactly that subtree — unless it contributed `topics`, the MQTT topic filters its
// work needs (a home-automation hub needs the devices' topics); then the role grants exactly those
// (topics.ts). A list that is not valid topic filters is refused, and the consumer is not created
// or changed until it is fixed.
//
// What a consumer is told (its binding): `at` — the broker's machine — and `port`, the machine port
// of the MQTT listener (the manifest's `serves`); `as` is its login, and its copy of the password is
// the pair credential the mesh delivers to it.
// role scoped to exactly that subtree.
//
// **The login and password are the mesh's, not the provisioner's (ADR 0048).** The mesh derives the
// login and hands it to both ends so they agree, and mints the password and delivers a copy to each.
@@ -22,7 +15,6 @@
import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner";
import { emit } from "@novox/mesh-sdk/events";
import { MosquittoClient } from "../client.js";
import { topicFilters } from "../topics.js";
const mosquitto = MosquittoClient.fromEnv();
@@ -37,20 +29,13 @@ async function announce(type: string, body: Record<string, string>): Promise<voi
runProvisioner("mqtt-topic", {
async create(p: Provision): Promise<void> {
// By default the consumer's own subtree, so one cannot read another's topics; what it
// contributed as `topics` otherwise.
const granted = topicFilters(p.values, p.as);
if ("problem" in granted) {
// Thrown, so the harness logs it and retries: the consumer stays as it was (or absent) until
// its contribution is valid, rather than being given a grant it did not ask for.
throw new Error(`${p.as}: ${granted.problem}`);
}
await mosquitto.createScopedClient(p.as, p.password, granted.filters);
// The topic subtree is scoped to the consumer's own login, so one cannot read another's topics.
const topicPrefix = p.as;
await mosquitto.createScopedClient(p.as, p.password, topicPrefix);
await announce("topic.provisioned", {
consumer: p.consumer ?? "",
username: p.as,
topicPrefix: granted.own ? p.as : "",
topics: granted.filters.join(" "),
topicPrefix,
});
},
@@ -61,9 +46,6 @@ runProvisioner("mqtt-topic", {
// Asked every minute by the harness: whether the backend still holds this consumer exactly as
// the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
async holds(p: Provision): Promise<boolean> {
const granted = topicFilters(p.values, p.as);
// An invalid list was never applied; create refuses it again, loudly, on every pass.
if ("problem" in granted) return false;
return mosquitto.holdsClient(p.as, p.password, granted.filters);
return mosquitto.holdsClient(p.as, p.password);
},
});
-72
View File
@@ -1,72 +0,0 @@
// What a consumer of mqtt-topic is granted (topics.ts): its own subtree unless it contributed
// `topics`; a contributed list is granted exactly, refused whole when it is not topic filters; and
// the role is brought to exactly the wanted ACLs — missing ones added, stale ones removed — read from
// `mosquitto_ctrl dynsec getRole` as eclipse-mosquitto 2.1.2 prints it.
import { test } from "node:test";
import assert from "node:assert/strict";
import { filterProblem, missingAcls, parseRoleAcls, staleAcls, topicFilters, wantedAcls } from "../topics.ts";
test("a consumer that contributed nothing gets its own subtree", () => {
assert.deepEqual(topicFilters({}, "mesh_ace_hass"), { ok: true, filters: ["mesh_ace_hass/#"], own: true });
assert.deepEqual(topicFilters(undefined, "x"), { ok: true, filters: ["x/#"], own: true });
// Settings merge into every contribution: keys that are not `topics` change nothing.
assert.deepEqual(topicFilters({ endpoints: { web: {} } }, "x"), { ok: true, filters: ["x/#"], own: true });
});
test("a contributed list is granted exactly, duplicates once", () => {
assert.deepEqual(topicFilters({ topics: ["#"] }, "x"), { ok: true, filters: ["#"], own: false });
assert.deepEqual(topicFilters({ topics: ["stat/+/POWER", "tele/#", "tele/#", "/octoprint/x"] }, "x"), {
ok: true,
filters: ["stat/+/POWER", "tele/#", "/octoprint/x"],
own: false,
});
});
test("a list that is not topic filters is refused whole", () => {
for (const topics of [[], "#", [""], ["a/#/b"], ["a#"], ["a/b+"], [42], ["a\u0000b"], {}]) {
const out = topicFilters({ topics } as Record<string, unknown>, "x");
assert.equal(out.ok, false, JSON.stringify(topics));
}
assert.equal(filterProblem("+/+/#"), undefined);
assert.equal(filterProblem("#"), undefined);
});
const GET_ROLE = `Warning: You are running mosquitto_ctrl without encryption.
This means all of the configuration changes you are making are visible on the network, including passwords.
Rolename: u1
ACLs: publishClientSend : allow : # (priority: 0)
subscribePattern : allow : u1/# (priority: 0)
publishClientReceive : deny : secret topic/with space (priority: -1)
`;
test("getRole's ACL lines are read, the warning and headings are not", () => {
assert.deepEqual(parseRoleAcls(GET_ROLE), [
{ type: "publishClientSend", allow: true, topic: "#" },
{ type: "subscribePattern", allow: true, topic: "u1/#" },
{ type: "publishClientReceive", allow: false, topic: "secret topic/with space" },
]);
assert.deepEqual(parseRoleAcls("Rolename: empty\nACLs:\n"), []);
});
test("the role is brought to exactly the wanted ACLs", () => {
const current = parseRoleAcls(GET_ROLE);
const wanted = wantedAcls(["u1/#"]);
assert.deepEqual(wanted, [
{ type: "publishClientSend", allow: true, topic: "u1/#" },
{ type: "publishClientReceive", allow: true, topic: "u1/#" },
{ type: "subscribePattern", allow: true, topic: "u1/#" },
]);
assert.deepEqual(missingAcls(current, wanted), [
{ type: "publishClientSend", allow: true, topic: "u1/#" },
{ type: "publishClientReceive", allow: true, topic: "u1/#" },
]);
assert.deepEqual(staleAcls(current, wanted), [
{ type: "publishClientSend", allow: true, topic: "#" },
{ type: "publishClientReceive", allow: false, topic: "secret topic/with space" },
]);
assert.deepEqual(staleAcls(wanted, wanted), []);
assert.deepEqual(missingAcls(wanted, wanted), []);
});
-107
View File
@@ -1,107 +0,0 @@
// Which topics a consumer of `mqtt-topic` may use — the one choice a consumer makes about its grant.
//
// **By default, its own subtree and nothing else.** A consumer connects as the login the mesh derived
// (`as`) and may publish, receive and subscribe under `<as>/#` — isolated from every other consumer,
// which is the point of a per-consumer client (novox/hq ADR 0039/0048).
//
// **A consumer whose work IS the shared topic space says so.** Home Assistant discovers devices
// under `homeassistant/#` and `tasmota/discovery/#` and follows whatever state topics they announce;
// Node-RED's flows subscribe to the topics devices publish on (`stat/<device>/POWER`, …). Confined
// to `<as>/#` neither could do its job. So a consumer contributes `topics` to its `mqtt-topic`
// requirement — a list of MQTT topic filters — and the provisioner grants exactly those, both ways.
// Because assignment settings merge into every contribution, an operator narrows (or widens) the
// list per machine with the same key, without editing a manifest.
//
// Pure, so it is tested without a broker (test/topics.test.ts).
/** The dynsec ACL types a granted filter carries: send to it, receive from it, subscribe to it. */
export const GRANTED_ACL_TYPES = ["publishClientSend", "publishClientReceive", "subscribePattern"] as const;
/** One ACL on a role, as `mosquitto_ctrl dynsec getRole` reports it. */
export interface Acl {
type: string;
allow: boolean;
topic: string;
}
export type Filters = { ok: true; filters: string[]; own: boolean } | { ok: false; problem: string };
/**
* The topic filters a consumer is granted: what it contributed as `topics`, or its own subtree when
* it contributed nothing. Refused — never silently narrowed or widened — when the list is not a
* list of valid MQTT topic filters: a grant that quietly differs from what was asked is a consumer
* that fails somewhere far from the cause.
*/
export function topicFilters(values: Readonly<Record<string, unknown>> | undefined, as: string): Filters {
const given = values?.topics;
if (given === undefined || given === null) {
return { ok: true, filters: [`${as}/#`], own: true };
}
if (!Array.isArray(given) || given.length === 0) {
return { ok: false, problem: `topics must be a non-empty list of MQTT topic filters, not ${JSON.stringify(given)}` };
}
const out: string[] = [];
for (const f of given) {
if (typeof f !== "string") {
return { ok: false, problem: `topics holds ${JSON.stringify(f)}, which is not a topic filter` };
}
const problem = filterProblem(f);
if (problem) return { ok: false, problem: `topic filter ${JSON.stringify(f)}: ${problem}` };
if (!out.includes(f)) out.push(f);
}
return { ok: true, filters: out, own: out.length === 1 && out[0] === `${as}/#` };
}
/** Why a string is not a valid MQTT topic filter (MQTT 3.1.1 §4.7), or undefined when it is one. */
export function filterProblem(filter: string): string | undefined {
if (filter.length === 0) return "it is empty";
if (Buffer.byteLength(filter, "utf8") > 65535) return "it is longer than MQTT allows";
if (filter.includes("\u0000")) return "it contains a NUL character";
const levels = filter.split("/");
for (let i = 0; i < levels.length; i++) {
const level = levels[i];
if (level.includes("#") && (level !== "#" || i !== levels.length - 1)) {
return "'#' must be a whole level, and the last one";
}
if (level.includes("+") && level !== "+") return "'+' must be a whole level";
}
return undefined;
}
/** The ACLs a role must carry to grant these filters: every granted type, allowed, on every filter. */
export function wantedAcls(filters: readonly string[]): Acl[] {
const out: Acl[] = [];
for (const topic of filters) {
for (const type of GRANTED_ACL_TYPES) out.push({ type, allow: true, topic });
}
return out;
}
/**
* The ACLs `mosquitto_ctrl dynsec getRole` lists, one per line under its "ACLs:" heading:
* `ACLs: publishClientSend : allow : # (priority: 0)`
* ` subscribePattern : allow : u1/# (priority: 0)`
*/
export function parseRoleAcls(output: string): Acl[] {
const out: Acl[] = [];
const line = /^(?:ACLs:)?\s*([A-Za-z]+)\s*:\s*(allow|deny)\s*:\s*(.*?)\s+\(priority:\s*-?\d+\)\s*$/;
for (const raw of output.split(/\r?\n/)) {
const m = raw.match(line);
if (m) out.push({ type: m[1], allow: m[2] === "allow", topic: m[3] });
}
return out;
}
const key = (a: Acl): string => `${a.type}\u0000${a.allow ? "allow" : "deny"}\u0000${a.topic}`;
/** ACLs a role carries that it should not: in `current` and not in `wanted`. */
export function staleAcls(current: readonly Acl[], wanted: readonly Acl[]): Acl[] {
const want = new Set(wanted.map(key));
return current.filter((a) => !want.has(key(a)));
}
/** ACLs a role should carry and does not. */
export function missingAcls(current: readonly Acl[], wanted: readonly Acl[]): Acl[] {
const have = new Set(current.map(key));
return wanted.filter((a) => !have.has(key(a)));
}
+1 -1
View File
@@ -8,5 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": ["topics.ts", "client.ts", "index.ts", "provisioner/index.ts", "tools/index.ts", "bootstrap/index.ts"]
"include": ["client.ts", "index.ts", "provisioner/index.ts", "tools/index.ts", "bootstrap/index.ts"]
}
+16 -15
View File
@@ -21,47 +21,48 @@
"listens": [
{
"name": "database",
"port": 1433,
"port": 4848,
"protocol": "tcp",
"from": "mesh",
"why": "modules on any machine that were granted a database, and the people who were given its address; the machine port is the assignment's to pin"
"why": "modules on any machine that were granted a database. 4848, not 1433: the machine this replaces has served it there since it was installed, and every consumer was handed that number"
}
],
"serves": {
"mssql-database": {}
},
"receives": {
"mssql-database": "${dir:grants}/mesh.json"
"mssql-database": "/var/lib/mssql/grants/mesh.json"
},
"grants": {
"mssql-database": "${dir:grants}"
"mssql-database": "/var/lib/mssql/grants"
},
"own-secrets": {
"sa": "${dir:state}/sa.secret",
"broker": "${dir:mesh-state}/broker"
"sa": "/var/lib/mssql/sa.secret",
"broker": "/var/lib/mesh/mssql/broker"
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/mssql",
"mode": "0700"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/mssql",
"mode": "0700"
},
{
"id": "grants",
"type": "directory",
"path": "/var/lib/mssql/grants",
"mode": "0700"
},
{
"id": "sa-env",
"type": "file",
"path": "${dir:state}/sa.env",
"path": "/var/lib/mssql/sa.env",
"mode": "0600",
"content": "ACCEPT_EULA=Y\nMSSQL_SA_PASSWORD=${secret:sa}\n"
},
@@ -83,7 +84,7 @@
"image": "mcr.microsoft.com/mssql/server@sha256:4402d880dd4c34bfa7d8705e56a86cd6c88da80a1f6bbbe741f999e76264a090",
"network": "mssql",
"env-file": [
"${dir:state}/sa.env"
"/var/lib/mssql/sa.env"
],
"ports": [
"1433"
@@ -99,9 +100,9 @@
"name": "mesh-mssql",
"network": "mssql",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:grants}:/var/lib/mssql/grants:ro",
"${dir:state}/sa.secret:/run/secrets/sa:ro"
"/var/lib/mesh/mssql/broker:/run/secrets/broker:ro",
"/var/lib/mssql/grants:/var/lib/mssql/grants:ro",
"/var/lib/mssql/sa.secret:/run/secrets/sa:ro"
],
"env": {
"MESH_PROVISION_MSSQL": "mssql://sa@mssql:1433/master",
+10 -10
View File
@@ -18,14 +18,14 @@
}
},
"binds": {
"postgres-database": "${dir:state}/database.json",
"route": "${dir:state}/route.json"
"postgres-database": "/var/lib/n8n/database.json",
"route": "/var/lib/n8n/route.json"
},
"secrets": {
"postgres-database": "${dir:state}/database.secret"
"postgres-database": "/var/lib/n8n/database.secret"
},
"own-secrets": {
"basic-auth": "${dir:state}/basic-auth.secret"
"basic-auth": "/var/lib/n8n/basic-auth.secret"
},
"listens": [
{
@@ -33,15 +33,15 @@
"port": 5682,
"protocol": "tcp",
"from": "mesh",
"why": "the n8n editor and webhook endpoints over http; its public name is a route grant, and route-proxy reaches it on this published port"
"why": "the n8n editor and webhook endpoints over http; the public name n8n.novox.be is a route grant, and route-proxy reaches it on this published port"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/n8n",
"mode": "0700"
},
{
"id": "data",
@@ -53,9 +53,9 @@
{
"id": "server-env",
"type": "file",
"path": "${dir:state}/server.env",
"path": "/var/lib/n8n/server.env",
"mode": "0600",
"content": "N8N_HOST=${bound:route:name}\nN8N_PORT=5678\nN8N_PROTOCOL=https\nWEBHOOK_URL=https://${bound:route:name}/\nN8N_BASIC_AUTH_ACTIVE=true\nN8N_BASIC_AUTH_USER=admin\nN8N_BASIC_AUTH_PASSWORD=${secret:basic-auth}\nNODE_FUNCTION_ALLOW_BUILTIN=*\nNODE_FUNCTION_ALLOW_EXTERNAL=*\nDB_TYPE=postgresdb\nDB_POSTGRESDB_HOST=${bound:postgres-database:at}\nDB_POSTGRESDB_PORT=${bound:postgres-database:port}\nDB_POSTGRESDB_DATABASE=${bound:postgres-database:as}\nDB_POSTGRESDB_USER=${bound:postgres-database:as}\nDB_POSTGRESDB_PASSWORD=${secret:postgres-database}\n"
"content": "N8N_HOST=n8n.novox.be\nN8N_PORT=5678\nN8N_PROTOCOL=https\nWEBHOOK_URL=https://n8n.novox.be/\nN8N_BASIC_AUTH_ACTIVE=true\nN8N_BASIC_AUTH_USER=admin\nN8N_BASIC_AUTH_PASSWORD=${secret:basic-auth}\nNODE_FUNCTION_ALLOW_BUILTIN=*\nNODE_FUNCTION_ALLOW_EXTERNAL=*\nDB_TYPE=postgresdb\nDB_POSTGRESDB_HOST=${bound:postgres-database:at}\nDB_POSTGRESDB_PORT=${bound:postgres-database:port}\nDB_POSTGRESDB_DATABASE=${bound:postgres-database:as}\nDB_POSTGRESDB_USER=${bound:postgres-database:as}\nDB_POSTGRESDB_PASSWORD=${secret:postgres-database}\n"
},
{
"id": "net",
@@ -69,7 +69,7 @@
"image": "n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0",
"network": "n8n",
"env-file": [
"${dir:state}/server.env"
"/var/lib/n8n/server.env"
],
"ports": [
"5678"
+1 -2
View File
@@ -62,14 +62,13 @@
"volumes": [
"/var/lib/mesh-broker-nats:/data",
"/var/lib/nats-module/conf:/etc/nats:ro",
"${access:tls}:/tls:ro"
"/var/lib/mesh-broker-tls:/tls:ro"
],
"artifact": "server"
}
],
"accesses": [
{
"id": "tls",
"path": "/var/lib/mesh-broker-tls",
"mode": "read"
}
+6 -6
View File
@@ -31,7 +31,7 @@
],
"own-secrets": {
"admin": "${dir:state}/admin.secret",
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/nextcloud/broker"
},
"capabilities": [
"container-runtime"
@@ -49,8 +49,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/nextcloud",
"mode": "0700"
},
{
"id": "state",
@@ -90,7 +90,7 @@
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/nextcloud/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -101,8 +101,8 @@
"name": "mesh-nextcloud",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"/var/lib/mesh/nextcloud/broker:/run/secrets/broker:ro",
"/var/lib/mesh/nextcloud/config.json:/run/config/config.json:ro",
"${dir:state}/admin.secret:/run/secrets/admin:ro",
"/var/run/docker.sock:/var/run/docker.sock"
],
+1 -4
View File
@@ -13,7 +13,7 @@ ARG RUNTIME_BASE
FROM ${BUILD_BASE} AS build
WORKDIR /app/modules/nodered
COPY . .
RUN node /app/node_modules/typescript/bin/tsc client.ts tools/index.ts mqtt/probe.ts mqtt/connection.ts mqtt/index.ts \
RUN node /app/node_modules/typescript/bin/tsc client.ts tools/index.ts \
--module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist
FROM ${RUNTIME_BASE}
@@ -22,6 +22,3 @@ COPY --from=build /app/modules/nodered/dist /app/modules/nodered/dist
# provider's provisioner runs its reconcile loop in the same process, with the broker connected —
# the convention novox/hq issues 060/061 settled.
ENV MESH_TOOL_MODULES=/app/modules/nodered/dist/tools/index.js
# NOT dist/mqtt/index.js: that is a step the host runs to completion, named by the `mqtt`
# container's args as `mesh-tools run …` (novox/hq ADR 0052). Listed here it would run inside the
# serving sidecar too, and exit it.
+3 -38
View File
@@ -3,8 +3,7 @@
//
// Node-RED exposes a runtime admin API under its base URL: GET/POST /flows for the whole flow
// configuration, GET /nodes for installed node modules. A default install has no auth; when
// adminAuth is on, a bearer token is required — the module's settings accept the mesh-minted
// api-token, which the runtime config file carries as `token`.
// adminAuth is on, a bearer token (minted at /auth/token) is required.
import { readFileSync } from "node:fs";
@@ -82,35 +81,6 @@ export class NodeRedClient {
return modules.map((m) => ({ name: m.name, version: m.version, types: m.types ?? [] }));
}
/** The whole flow configuration with its revision (API v2), for a deploy that must not clobber
* a change made meanwhile. */
async flowsWithRev(): Promise<{ rev: string; flows: any[] }> {
const body = await this.req("/flows", { headers: this.headers({ "Node-RED-API-Version": "v2" }) });
return { rev: String(body?.rev ?? ""), flows: Array.isArray(body?.flows) ? body.flows : [] };
}
/** A node's stored credentials as Node-RED shows them: plain fields, and `has_<field>` for secret ones. */
async credentials(type: string, id: string): Promise<{ user?: string; has_password?: boolean }> {
return (await this.req(`/credentials/${encodeURIComponent(type)}/${encodeURIComponent(id)}`, { headers: this.headers() })) ?? {};
}
/**
* Deploy the flow configuration read at `rev`. Node-RED answers 409 when the flows changed since,
* rather than overwriting what someone deployed in between. A node carrying `credentials` has them
* stored (encrypted) and counts as changed, so a "nodes" deploy restarts it and nothing else.
*/
async deployFlowsAt(rev: string, config: any[], type = "nodes"): Promise<void> {
await this.req("/flows", {
method: "POST",
headers: this.headers({
"Content-Type": "application/json",
"Node-RED-API-Version": "v2",
"Node-RED-Deployment-Type": type,
}),
body: JSON.stringify({ rev, flows: config }),
});
}
/**
* Replace the whole flow configuration and deploy. Returns the new revision. `type` maps to
* Node-RED's deployment types — "full" (default), "nodes", or "flows".
@@ -118,13 +88,8 @@ export class NodeRedClient {
async deployFlows(config: any[], type = "full"): Promise<{ rev?: string; nodeCount: number }> {
const body = await this.req("/flows", {
method: "POST",
// v2 answers { rev }; v1 answers 204 with no body, which req() cannot parse.
headers: this.headers({
"Content-Type": "application/json",
"Node-RED-API-Version": "v2",
"Node-RED-Deployment-Type": type,
}),
body: JSON.stringify({ flows: config }),
headers: this.headers({ "Content-Type": "application/json", "Node-RED-Deployment-Type": type }),
body: JSON.stringify(config),
});
return { rev: body?.rev, nodeCount: config.length };
}
+13 -91
View File
@@ -5,9 +5,7 @@
"flows.deployed"
],
"own-secrets": {
"admin": "${dir:mesh-state}/admin",
"api-token": "${dir:mesh-state}/api-token",
"broker": "${dir:mesh-state}/broker"
"broker": "/var/lib/mesh/nodered/broker"
},
"capabilities": [
"container-runtime"
@@ -25,74 +23,38 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/mesh/nodered",
"mode": "0700"
},
{
"id": "data",
"type": "directory",
"path": "/services/nodered/data",
"mode": "0700",
"owner": "1000:1000"
},
{
"id": "written",
"type": "directory",
"mode": "0700"
},
{
"id": "settings-code",
"type": "file",
"path": "${dir:state}/settings.js",
"mode": "0600",
"owner": "1000:1000",
"content": "// Node-RED's settings, written by the mesh from the nodered module. What an assignment may change\n// is settings.json beside this file (merged key by key); the credentials are the mesh's secrets and\n// reach Node-RED only through this file. The flows' own credentials stay encrypted in the user\n// directory under the key Node-RED keeps there (.config.runtime.json), which is data, not this.\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst crypto = require(\"crypto\");\n\nconst ADMIN_PASSWORD = \"${secret:admin}\";\nconst API_TOKEN = \"${secret:api-token}\";\nconst ADMIN = { username: \"admin\", permissions: \"*\" };\n\nconst settings = JSON.parse(fs.readFileSync(path.join(__dirname, \"settings.json\"), \"utf8\"));\n// The mesh's keys, not Node-RED's: endpoints lands in every merged file; timeZone is the\n// assignment's way to set the zone flows schedule and format in; mqtt names the broker nodes the\n// module's MQTT step keeps pointed at the mesh's broker; topics is what nodered asks the broker for.\nif (settings.timeZone) process.env.TZ = settings.timeZone;\ndelete settings.timeZone;\ndelete settings.endpoints;\ndelete settings.mqtt;\ndelete settings.topics;\n\nfunction same(a, b) {\n const x = crypto.createHash(\"sha256\").update(String(a)).digest();\n const y = crypto.createHash(\"sha256\").update(String(b)).digest();\n return crypto.timingSafeEqual(x, y);\n}\n\n// The admin secret is a password, or, accepted from an existing install, the bcrypt hash its\n// settings held, so the password people already use keeps working.\nfunction passwordMatches(given) {\n if (/^\\$2[aby]\\$\\d\\d\\$/.test(ADMIN_PASSWORD)) return require(\"bcryptjs\").compare(String(given), ADMIN_PASSWORD);\n return Promise.resolve(same(given, ADMIN_PASSWORD));\n}\n\nmodule.exports = Object.assign(settings, {\n uiPort: 1880,\n adminAuth: {\n type: \"credentials\",\n users: (username) => Promise.resolve(username === ADMIN.username ? ADMIN : null),\n authenticate: (username, password) =>\n username === ADMIN.username\n ? passwordMatches(password).then((ok) => (ok ? ADMIN : null))\n : Promise.resolve(null),\n // The module's own tools call the admin API with this bearer token.\n tokens: (token) => Promise.resolve(same(token, API_TOKEN) ? { username: \"mesh\", permissions: \"*\" } : null),\n },\n});\n"
},
{
"id": "settings",
"type": "file",
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "1000:1000",
"merge": "json",
"content": "{\n \"flowFile\": \"flows.json\",\n \"flowFilePretty\": true,\n \"diagnostics\": { \"enabled\": true, \"ui\": true },\n \"runtimeState\": { \"enabled\": false, \"ui\": false },\n \"logging\": { \"console\": { \"level\": \"info\", \"metrics\": false, \"audit\": false } },\n \"exportGlobalContextKeys\": false,\n \"externalModules\": {},\n \"editorTheme\": { \"projects\": { \"enabled\": false } },\n \"functionExternalModules\": true,\n \"debugMaxLength\": 1000,\n \"mqttReconnectTime\": 15000,\n \"serialReconnectTime\": 15000\n}\n"
},
{
"id": "server",
"type": "container",
"name": "nodered",
"image": "nodered/node-red@sha256:a649dd711d55490151a2c39a8e48ad0c44325488fbc0e66315f2d2e19e5e1ace",
"image": "nodered/node-red@sha256:02a2b92a41b73d2bc388238b86e4fcaab7fb5466373adb24e1df6aa5845265ff",
"env": {
"TZ": "Etc/UTC"
},
"ports": [
"1880"
],
"args": [
"--settings",
"/data/settings.js"
],
"volumes": [
"${dir:data}:/data",
"${dir:state}/settings.js:/data/settings.js:ro",
"${dir:state}/settings.json:/data/settings.json:ro"
],
"restart-on": [
"settings-code",
"settings"
"/services/nodered/data:/data"
]
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/nodered/config.json",
"mode": "0600",
"content": "{\n \"token\": \"${secret:api-token}\"\n}\n"
"content": "{}\n",
"merge": "json"
},
{
"id": "runtime",
@@ -100,71 +62,31 @@
"name": "mesh-nodered",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro"
"/var/lib/mesh/nodered/broker:/run/secrets/broker:ro",
"/var/lib/mesh/nodered/config.json:/run/config/config.json:ro"
],
"env": {
"MESH_BROKER_FILE": "/run/secrets/broker",
"MESH_NODERED_URL": "http://127.0.0.1:${port:1880}",
"MESH_NODERED_URL": "http://127.0.0.1:1880",
"MESH_NODERED_CONFIG_FILE": "/run/config/config.json"
},
"restart-on": [
"runtime-config"
],
"artifact": "runtime"
},
{
"id": "mqtt",
"type": "container",
"name": "mesh-nodered-mqtt",
"network": "host",
"run-once": true,
"volumes": [
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"${dir:written}:/var/lib/nodered-provisions",
"${dir:state}/mqtt-topic.json:/run/provisions/mqtt-topic.json:ro",
"${dir:state}/mqtt-topic.secret:/run/provisions/mqtt-topic.secret:ro",
"${dir:state}/settings.json:/run/provisions/settings.json:ro"
],
"env": {
"MESH_NODERED_URL": "http://127.0.0.1:${port:1880}",
"MESH_NODERED_CONFIG_FILE": "/run/config/config.json",
"MESH_PROVISIONS_DIR": "/run/provisions",
"MESH_WRITTEN_DIR": "/var/lib/nodered-provisions"
},
"args": [
"run",
"/app/modules/nodered/dist/mqtt/index.js"
],
"restart-on": [
"bound-mqtt-topic",
"secret-mqtt-topic",
"settings"
],
"artifact": "runtime"
}
],
"requires": [
"mqtt-topic",
"route"
],
"contributes": {
"mqtt-topic": {
"topics": [
"#"
]
},
"route": {
"label": "nodered",
"endpoint": "web"
}
},
"binds": {
"route": "${dir:state}/route.json",
"mqtt-topic": "${dir:state}/mqtt-topic.json"
},
"secrets": {
"mqtt-topic": "${dir:state}/mqtt-topic.secret"
"route": "/var/lib/mesh/nodered/route.json"
},
"build": {
"on": [
-232
View File
@@ -1,232 +0,0 @@
// Node-RED's MQTT broker config node, pointed at the broker the mesh bound — `mqtt-topic`.
//
// **Why a step.** Node-RED keeps a broker as a config node in its flows (`flows.json`) and the login
// and password in its encrypted credentials file, both of them Node-RED's to write. So this reads the
// binding and the pair credential and makes the broker node say the same thing through Node-RED's
// admin API — `GET /flows`, then `POST /flows` with the changed node and its `credentials`, deployed
// as "nodes" so only what changed restarts — with the module's own `api-token`.
//
// **Which broker nodes are the mesh's.** Never guessed: a flow may talk to a broker that has nothing
// to do with this mesh. The step owns the node it creates itself (id `mesh-mqtt-topic`, "mesh:
// mqtt-topic") and the ones an assignment names in settings (`mqtt.brokers`: node ids — how ace's
// existing broker node, which every one of its MQTT flows uses, is handed over). With none named and
// none made yet, it makes one, so a fresh Node-RED has a broker the flows can pick.
//
// **Only the connection, and only when it differs.** Host, port, TLS off (the broker serves plain
// MQTT), login, password. Every other field of the node — client id, keepalive, birth/close/will
// messages — is left as it is. Node-RED never hands a stored password back, so the step keeps a
// digest of what it last wrote: equal host/port/login and an equal digest is "already as the mesh
// says".
//
// **Nothing loses its connection without someone seeing it.** The broker is asked first whether it
// takes the delivered login; if not, nothing is written and the step fails saying why.
//
// Pure logic over two seams (Node-RED, the broker), tested against fakes (test/mqtt.test.ts).
import { createHash } from "node:crypto";
import type { Probe } from "./probe.js";
export const PROVISION = "mqtt-topic";
/** The id and name of the broker node the step makes when none is named. */
export const MESH_BROKER_ID = "mesh-mqtt-topic";
export const MESH_BROKER_NAME = "mesh: mqtt-topic";
/** What the mesh wrote at `binds.mqtt-topic`. */
export interface Binding {
provision?: string;
from?: string;
at?: string;
as?: string;
serves?: Record<string, unknown>;
}
export type Outcome =
| { what: string; result: "unchanged"; note?: string }
| { what: string; result: "written"; fields: string[]; note?: string }
| { what: string; result: "refused"; problem: string };
/** A flow node; a broker config node carries `broker`, `port`, `usetls`. */
export interface FlowNode {
id: string;
type: string;
[key: string]: unknown;
}
/** Node-RED's admin API, as the step uses it. */
export interface NodeRed {
/** The whole flow configuration and its revision (API v2). */
flows(): Promise<{ rev: string; flows: FlowNode[] }>;
/** A node's stored credentials as Node-RED shows them: the user, and only whether a password is set. */
credentials(type: string, id: string): Promise<{ user?: string; has_password?: boolean }>;
/** Deploy the configuration against the revision it was read at; "nodes" restarts only what changed. */
deploy(rev: string, flows: FlowNode[]): Promise<void>;
}
export interface Marks {
get(name: string): Promise<string | undefined>;
set(name: string, digest: string): Promise<void>;
}
export interface Deps {
nodered: NodeRed;
probe: Probe;
marks: Marks;
}
export interface Wanted {
host: string;
port: number;
user: string;
password: string;
}
export function digest(...parts: (string | number)[]): string {
return createHash("sha256").update(parts.map(String).join("\u0000")).digest("hex");
}
function isLoopback(host: string): boolean {
const h = host.toLowerCase();
return h === "localhost" || h === "::1" || h === "[::1]" || /^127\./.test(h);
}
/**
* The broker and login the mesh says Node-RED uses. A loopback `at` — what the mesh hands a machine
* that is not on the private network — is refused: from Node-RED's own container it is Node-RED.
*/
export function wanted(binding: Binding | undefined, credential: string | undefined): { ok: true; want: Wanted } | { ok: false; problem: string } {
if (!binding) return { ok: false, problem: `no binding for ${PROVISION} was delivered — the mesh writes it before this step runs` };
const host = typeof binding.at === "string" ? binding.at.trim() : "";
if (!host) return { ok: false, problem: `the ${PROVISION} binding names no host (at)` };
if (isLoopback(host)) {
return {
ok: false,
problem:
`the ${PROVISION} binding says the broker is at ${host}, which from Node-RED's own container is Node-RED ` +
`itself; put the machine on the private network so the broker has an address Node-RED can dial`,
};
}
const port = Number(binding.serves?.port);
if (!Number.isInteger(port) || port <= 0 || port > 65535) {
return { ok: false, problem: `the ${PROVISION} binding serves no usable port (${String(binding.serves?.port)})` };
}
const scheme = binding.serves?.scheme;
if (scheme !== undefined && scheme !== "mqtt") return { ok: false, problem: `the ${PROVISION} binding serves scheme ${String(scheme)}; this step writes plain MQTT` };
const user = typeof binding.as === "string" ? binding.as.trim() : "";
if (!user) return { ok: false, problem: `the ${PROVISION} binding names no login (as)` };
const password = (credential ?? "").replace(/\n$/, "");
if (!password) return { ok: false, problem: `the ${PROVISION} credential is empty or was not delivered` };
return { ok: true, want: { host, port, user, password } };
}
/** The broker node ids an assignment named in settings (`mqtt.brokers`), or none. */
export function namedBrokers(settings: unknown): string[] {
const brokers = (settings as { mqtt?: { brokers?: unknown } } | undefined)?.mqtt?.brokers;
return Array.isArray(brokers) ? brokers.filter((b): b is string => typeof b === "string" && b.length > 0) : [];
}
/** A new broker node, Node-RED 5's defaults, pointed at the broker. */
export function newBrokerNode(want: Wanted): FlowNode {
return {
id: MESH_BROKER_ID, type: "mqtt-broker", name: MESH_BROKER_NAME,
broker: want.host, port: String(want.port), clientid: "", autoConnect: true, usetls: false,
protocolVersion: "4", keepalive: "60", cleansession: true, autoUnsubscribe: true,
birthTopic: "", birthQos: "0", birthRetain: "false", birthPayload: "", birthMsg: {},
closeTopic: "", closeQos: "0", closeRetain: "false", closePayload: "", closeMsg: {},
willTopic: "", willQos: "0", willRetain: "false", willPayload: "", willMsg: {},
userProps: "", sessionExpiry: "",
};
}
const markFor = (id: string, w: Wanted): string => digest("nodered-mqtt", id, w.host, w.port, w.user, w.password);
function scrub(err: unknown, secret: string): string {
let text = err instanceof Error ? err.message : String(err);
for (const form of new Set([secret, encodeURIComponent(secret)])) text = text.split(form).join("***");
return text;
}
/**
* Bring the mesh's broker nodes in line with the binding: one outcome per node. Never throws. A node
* the settings name that is not in the flows is refused (the others are still put right).
*/
export async function reconcileBrokers(deps: Deps, binding: Binding | undefined, credential: string | undefined, named: readonly string[]): Promise<Outcome[]> {
const w = wanted(binding, credential);
if ("problem" in w) return [{ what: "mqtt", result: "refused", problem: w.problem }];
const want = w.want;
let note: string | undefined;
try {
const probe = await deps.probe(want.host, want.port, want.user, want.password, "#");
if (probe.connack === 4 || probe.connack === 5) {
return [{
what: "mqtt",
result: "refused",
problem:
`the broker at ${want.host}:${want.port} does not (yet) take the login ${want.user} with the delivered password ` +
`(CONNACK ${probe.connack}); mosquitto's provisioner creates it from the grant — nothing was written`,
}];
}
if (probe.connack !== 0) return [{ what: "mqtt", result: "refused", problem: `the broker at ${want.host}:${want.port} answered CONNACK ${probe.connack}; nothing was written` }];
if (probe.suback === 0x80) note = `warning: ${want.user} may not subscribe to every topic; flows subscribing outside its grant will get nothing`;
} catch (err) {
return [{ what: "mqtt", result: "refused", problem: `the broker at ${want.host}:${want.port} could not be asked: ${scrub(err, want.password)}; nothing was written` }];
}
const outcomes: Outcome[] = [];
for (let attempt = 0; attempt < 2; attempt++) {
outcomes.length = 0;
try {
const { rev, flows } = await deps.nodered.flows();
const targets = named.length > 0 ? [...named] : [MESH_BROKER_ID];
const changed: { id: string; fields: string[] }[] = [];
for (const id of targets) {
let node = flows.find((n) => n.id === id);
if (node && node.type !== "mqtt-broker") {
outcomes.push({ what: `broker ${id}`, result: "refused", problem: `node ${id} is a ${node.type}, not an mqtt-broker` });
continue;
}
if (!node) {
if (id !== MESH_BROKER_ID) {
outcomes.push({ what: `broker ${id}`, result: "refused", problem: `the settings name broker node ${id}, and Node-RED's flows have no such node` });
continue;
}
node = newBrokerNode(want);
flows.push(node);
node.credentials = { user: want.user, password: want.password };
changed.push({ id, fields: ["node"] });
continue;
}
const fields: string[] = [];
if (String(node.broker ?? "") !== want.host) fields.push("broker");
if (Number(node.port ?? 0) !== want.port) fields.push("port");
if (node.usetls === true) fields.push("usetls");
const creds = await deps.nodered.credentials("mqtt-broker", id);
if ((creds.user ?? "") !== want.user) fields.push("user");
if (!creds.has_password || (await deps.marks.get(`broker-${id}`)) !== markFor(id, want)) fields.push("password");
if (fields.length === 0) {
outcomes.push(note ? { what: `broker ${id}`, result: "unchanged", note } : { what: `broker ${id}`, result: "unchanged" });
continue;
}
node.broker = want.host;
node.port = String(want.port);
node.usetls = false;
node.credentials = { user: want.user, password: want.password };
changed.push({ id, fields });
}
if (changed.length > 0) {
await deps.nodered.deploy(rev, flows);
for (const c of changed) {
await deps.marks.set(`broker-${c.id}`, markFor(c.id, want));
outcomes.push({ what: `broker ${c.id}`, result: "written", fields: c.fields, ...(note ? { note } : {}) });
}
}
return outcomes;
} catch (err) {
// A deploy against a revision someone else changed meanwhile (409) is read again once.
if (attempt === 0 && /\b409\b/.test(String(err))) continue;
return [...outcomes, { what: "mqtt", result: "refused", problem: scrub(err, want.password) }];
}
}
return outcomes;
}
-102
View File
@@ -1,102 +0,0 @@
// nodered's MQTT step — run once by the host after Node-RED starts, and again whenever the
// `mqtt-topic` binding, its pair credential or the settings change (the container's `restart-on`,
// novox/hq ADR 0099). It points the mesh's broker config nodes at the broker the mesh bound, through
// Node-RED's admin API (connection.ts). It connects to no mesh broker.
//
// Exits non-zero when anything could not be put right, so the node reports the step failed and the
// host runs it again on the next apply. Declared last in the manifest, so its failing gates nothing
// else of nodered's (novox/hq ADR 0136). Never prints a password.
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { NodeRedClient } from "../client.js";
import { namedBrokers, reconcileBrokers, type Binding, type Marks } from "./connection.js";
import { probeBroker } from "./probe.js";
const dir = process.env.MESH_PROVISIONS_DIR ?? "/run/provisions";
const writtenDir = process.env.MESH_WRITTEN_DIR ?? "/var/lib/nodered-provisions";
const waitSeconds = Number(process.env.MESH_NODERED_WAIT_SECONDS ?? "180");
const readIfThere = (path: string): Promise<string | undefined> => readFile(path, "utf8").catch(() => undefined);
const parse = <T>(raw: string | undefined): T | undefined => {
if (raw === undefined) return undefined;
try {
return JSON.parse(raw) as T;
} catch {
return undefined;
}
};
const marks: Marks = {
async get(name) {
return (await readIfThere(join(writtenDir, `${name}.digest`)))?.trim() || undefined;
},
async set(name, value) {
await mkdir(writtenDir, { recursive: true, mode: 0o700 });
const path = join(writtenDir, `${name}.digest`);
await writeFile(`${path}.tmp`, `${value}\n`, { mode: 0o600 });
await rename(`${path}.tmp`, path);
},
};
let client: NodeRedClient;
try {
client = NodeRedClient.fromEnv();
} catch (err) {
console.error(`[nodered-mqtt] ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
}
/** Node-RED answers the admin API once its flows are loaded and the token is good. */
async function ready(): Promise<boolean> {
const until = Date.now() + waitSeconds * 1000;
for (;;) {
try {
await client.flowsWithRev();
return true;
} catch (err) {
if (/\b(401|403)\b/.test(String(err))) {
console.error("[nodered-mqtt] Node-RED refuses the api-token — settings.js and this step disagree");
return false;
}
}
if (Date.now() >= until) return false;
await new Promise((r) => setTimeout(r, 2000));
}
}
if (!(await ready())) {
console.error(`[nodered-mqtt] Node-RED's admin API did not answer at ${client.baseUrl} within ${waitSeconds}s`);
process.exit(1);
}
const binding = parse<Binding>(await readIfThere(join(dir, "mqtt-topic.json")));
const secret = await readIfThere(join(dir, "mqtt-topic.secret"));
const settings = parse<unknown>(await readIfThere(join(dir, "settings.json")));
const outcomes = await reconcileBrokers(
{
nodered: {
flows: () => client.flowsWithRev(),
credentials: (type, id) => client.credentials(type, id),
deploy: (rev, flows) => client.deployFlowsAt(rev, flows, "nodes"),
},
probe: probeBroker,
marks,
},
binding,
secret,
namedBrokers(settings),
);
let failed = 0;
for (const o of outcomes) {
if (o.result === "unchanged") console.log(`[nodered-mqtt] ${o.what}: already as the mesh says${o.note ? ` — ${o.note}` : ""}`);
else if (o.result === "written") console.log(`[nodered-mqtt] ${o.what}: wrote ${o.fields.join(", ")}${o.note ? ` — ${o.note}` : ""}`);
else {
failed++;
console.error(`[nodered-mqtt] ${o.what}: ${o.problem}`);
}
}
process.exitCode = failed > 0 ? 1 : 0;
-117
View File
@@ -1,117 +0,0 @@
// Ask the broker, before Node-RED is told anything, whether it takes the login and password the
// mesh delivered — and whether that login may subscribe to every topic, as flows expect.
//
// One MQTT 3.1.1 session: CONNECT (clean, a throwaway client id, so no flow's session is taken
// over), read the CONNACK, optionally SUBSCRIBE once and read the SUBACK, DISCONNECT. No dependency:
// the handful of bytes MQTT needs for this are written here.
import { randomBytes } from "node:crypto";
import { connect } from "node:net";
export interface ProbeResult {
/** 0 accepted; 4 bad username or password; 5 not authorised. */
connack: number;
/** The SUBACK return code for the filter asked about: 0–2 granted, 0x80 refused. */
suback?: number;
}
export type Probe = (host: string, port: number, username: string, password: string, subscribe?: string) => Promise<ProbeResult>;
function str(v: string): Buffer {
const b = Buffer.from(v, "utf8");
const len = Buffer.alloc(2);
len.writeUInt16BE(b.length);
return Buffer.concat([len, b]);
}
function packet(type: number, body: Buffer): Buffer {
let remaining = body.length;
const lenBytes: number[] = [];
do {
let byte = remaining % 128;
remaining = Math.floor(remaining / 128);
if (remaining > 0) byte |= 0x80;
lenBytes.push(byte);
} while (remaining > 0);
return Buffer.concat([Buffer.from([type, ...lenBytes]), body]);
}
/** The first complete packet in `buf`: its type byte, its body, and how many bytes it took. */
export function firstPacket(buf: Buffer): { type: number; body: Buffer; used: number } | undefined {
if (buf.length < 2) return undefined;
let length = 0;
let multiplier = 1;
let i = 1;
for (;;) {
if (i >= buf.length) return undefined;
const byte = buf[i++];
length += (byte & 0x7f) * multiplier;
if ((byte & 0x80) === 0) break;
multiplier *= 128;
if (i > 4) throw new Error("malformed MQTT remaining length");
}
if (buf.length < i + length) return undefined;
return { type: buf[0], body: buf.subarray(i, i + length), used: i + length };
}
export const probeBroker: Probe = (host, port, username, password, subscribe) => {
const connectBody = Buffer.concat([
str("MQTT"),
Buffer.from([4, 0xc2, 0, 10]), // level 4 (3.1.1); username + password + clean session; keepalive 10s
str(`mesh-probe-${randomBytes(6).toString("hex")}`),
str(username),
str(password),
]);
return new Promise((resolve, reject) => {
const socket = connect({ host, port });
let buf = Buffer.alloc(0);
const result: ProbeResult = { connack: -1 };
const timer = setTimeout(() => {
socket.destroy();
reject(new Error(`no answer from the broker at ${host}:${port} within 10s`));
}, 10_000);
const finish = (): void => {
clearTimeout(timer);
if (result.connack === 0) socket.end(Buffer.from([0xe0, 0]));
else socket.destroy();
resolve(result);
};
socket.on("connect", () => socket.write(packet(0x10, connectBody)));
socket.on("data", (chunk) => {
buf = Buffer.concat([buf, chunk]);
for (;;) {
let p;
try {
p = firstPacket(buf);
} catch (err) {
clearTimeout(timer);
socket.destroy();
reject(err);
return;
}
if (!p) return;
buf = buf.subarray(p.used);
const kind = p.type >> 4;
if (kind === 2) {
result.connack = p.body[1] ?? -1;
if (result.connack !== 0 || !subscribe) return finish();
// SUBSCRIBE, packet id 1, one filter at QoS 0.
socket.write(packet(0x82, Buffer.concat([Buffer.from([0, 1]), str(subscribe), Buffer.from([0])])));
} else if (kind === 9) {
result.suback = p.body[2];
return finish();
}
}
});
socket.on("error", (err) => {
clearTimeout(timer);
reject(err);
});
socket.on("close", () => {
if (result.connack === -1) {
clearTimeout(timer);
reject(new Error(`the broker at ${host}:${port} closed the connection without answering`));
}
});
});
};
+1 -6
View File
@@ -1,14 +1,9 @@
{
"name": "@novox/module-nodered",
"version": "0.1.0",
"description": "nodered \u2014 flow-based automation. Its client and tools live here (novox/hq ADR 0039).",
"description": "nodered — flow-based automation. Its client and tools live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"scripts": {
"build": "tsc client.ts tools/index.ts mqtt/probe.ts mqtt/connection.ts mqtt/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
"typecheck": "tsc -p tsconfig.json",
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
},
"dependencies": {
"@novox/mesh-sdk": "^0.1.0"
},
-124
View File
@@ -1,124 +0,0 @@
// What holds nodered's MQTT step (mqtt/connection.ts): the broker nodes the mesh owns — the one it
// makes, or the ones settings name — are made to use the broker and login the mesh bound, only after
// the broker takes that login; every other field of a node is kept; nothing is deployed when nothing
// differs; a node that is not named is never touched; a loopback broker address is refused.
//
// Node-RED and the broker are fakes answering as the real ones do (admin API v2 of nodered/node-red
// 5.0.7, the build ace runs).
import { test } from "node:test";
import assert from "node:assert/strict";
import { MESH_BROKER_ID, namedBrokers, reconcileBrokers, type Binding, type FlowNode, type Marks, type NodeRed } from "../mqtt/connection.ts";
import type { Probe } from "../mqtt/probe.ts";
const MINTED = "mesh-minted-password";
const binding = (at = "ace.internal"): Binding => ({ provision: "mqtt-topic", from: "ace", at, as: "mesh_ace_nodered", serves: { scheme: "mqtt", port: 1883 } });
/** ace's flows, reduced: its one broker node (dead, zurag.be:1884) and a node that uses it. */
function aceFlows(): FlowNode[] {
return [
{ id: "2b0aece9c5f3b307", type: "mqtt-broker", name: "MQTT Broker", broker: "zurag.be", port: "1884", clientid: "", usetls: false, protocolVersion: "4", keepalive: "60" },
{ id: "fe0cae96f1e3ae4d", type: "mqtt in", topic: "stat/sonoff_office_light_switch/RESULT", broker: "2b0aece9c5f3b307", z: "t" },
{ id: "other-broker", type: "mqtt-broker", name: "someone else's", broker: "test.mosquitto.org", port: "1883" },
];
}
function fakeNodeRed(flows: FlowNode[], creds: Record<string, { user?: string; password?: string }> = {}) {
let rev = "r1";
const deploys: FlowNode[][] = [];
const nodered: NodeRed = {
async flows() {
return { rev, flows: structuredClone(flows) };
},
async credentials(_type, id) {
const c = creds[id] ?? {};
return { user: c.user, has_password: Boolean(c.password) };
},
async deploy(at, next) {
if (at !== rev) throw new Error("Node-RED /flows: 409 version_mismatch");
for (const n of next) {
if (n.credentials) creds[n.id] = { ...(creds[n.id] ?? {}), ...(n.credentials as object) };
}
flows.splice(0, flows.length, ...next.map(({ credentials: _c, ...n }) => n as FlowNode));
deploys.push(next);
rev = `r${deploys.length + 1}`;
},
};
return { nodered, deploys, flows, creds };
}
const marks = (): Marks & { store: Map<string, string> } => {
const store = new Map<string, string>();
return { store, get: async (k) => store.get(k), set: async (k, v) => void store.set(k, v) };
};
const takes: Probe = async (_h, _p, user, pass) => ({ connack: user === "mesh_ace_nodered" && pass === MINTED ? 0 : 5, suback: 0 });
test("ace: the named broker node is moved to the bound broker and login; the other broker is not touched", async () => {
const f = fakeNodeRed(aceFlows(), { "2b0aece9c5f3b307": { user: "luffy", password: "old" }, "other-broker": { user: "x", password: "y" } });
const m = marks();
const out = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: m }, binding(), `${MINTED}\n`, ["2b0aece9c5f3b307"]);
assert.deepEqual(out, [{ what: "broker 2b0aece9c5f3b307", result: "written", fields: ["broker", "port", "user", "password"] }]);
const node = f.flows.find((n) => n.id === "2b0aece9c5f3b307");
assert.deepEqual(node, { ...aceFlows()[0], broker: "ace.internal", port: "1883", usetls: false });
assert.deepEqual(f.creds["2b0aece9c5f3b307"], { user: "mesh_ace_nodered", password: MINTED });
assert.deepEqual(f.flows.find((n) => n.id === "other-broker"), aceFlows()[2]);
assert.deepEqual(f.creds["other-broker"], { user: "x", password: "y" });
// Only the changed node carried credentials in the deploy.
assert.deepEqual(f.deploys[0].filter((n) => n.credentials).map((n) => n.id), ["2b0aece9c5f3b307"]);
// Again: nothing differs, nothing is deployed.
const again = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: m }, binding(), MINTED, ["2b0aece9c5f3b307"]);
assert.deepEqual(again, [{ what: "broker 2b0aece9c5f3b307", result: "unchanged" }]);
assert.equal(f.deploys.length, 1);
});
test("fresh: with nothing named, the step makes its own broker node", async () => {
const f = fakeNodeRed([]);
const out = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: marks() }, binding(), MINTED, []);
assert.deepEqual(out, [{ what: `broker ${MESH_BROKER_ID}`, result: "written", fields: ["node"] }]);
assert.equal(f.flows[0].type, "mqtt-broker");
assert.equal(f.flows[0].broker, "ace.internal");
assert.deepEqual(f.creds[MESH_BROKER_ID], { user: "mesh_ace_nodered", password: MINTED });
});
test("a login the broker does not take is never written", async () => {
const f = fakeNodeRed(aceFlows());
const out = await reconcileBrokers({ nodered: f.nodered, probe: async () => ({ connack: 5 }), marks: marks() }, binding(), MINTED, ["2b0aece9c5f3b307"]);
assert.equal(out[0].result, "refused");
assert.equal(f.deploys.length, 0);
});
test("a named node that is not there, or a loopback broker, is refused", async () => {
const f = fakeNodeRed(aceFlows());
const out = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: marks() }, binding(), MINTED, ["gone"]);
assert.match((out[0] as { problem: string }).problem, /no such node/);
const lo = await reconcileBrokers({ nodered: f.nodered, probe: takes, marks: marks() }, binding("127.0.0.1"), MINTED, []);
assert.match((lo[0] as { problem: string }).problem, /Node-RED itself/);
assert.equal(f.deploys.length, 0);
});
test("a deploy that raced another is read again once", async () => {
const f = fakeNodeRed(aceFlows());
let first = true;
const racing: NodeRed = {
...f.nodered,
async flows() {
const got = await f.nodered.flows();
if (first) {
first = false;
return { ...got, rev: "stale" };
}
return got;
},
};
const out = await reconcileBrokers({ nodered: racing, probe: takes, marks: marks() }, binding(), MINTED, ["2b0aece9c5f3b307"]);
assert.equal(out[0].result, "written");
assert.equal(f.deploys.length, 1);
});
test("settings name broker nodes under mqtt.brokers", () => {
assert.deepEqual(namedBrokers({ mqtt: { brokers: ["a", "", 3, "b"] } }), ["a", "b"]);
assert.deepEqual(namedBrokers({ endpoints: {} }), []);
assert.deepEqual(namedBrokers(undefined), []);
});
+1 -7
View File
@@ -8,11 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": [
"client.ts",
"tools/index.ts",
"mqtt/probe.ts",
"mqtt/connection.ts",
"mqtt/index.ts"
]
"include": ["client.ts", "tools/index.ts"]
}
@@ -1,5 +1,5 @@
{
"module": "website",
"module": "novox.be",
"version": "1",
"capabilities": [
"container-runtime"
@@ -14,41 +14,38 @@
}
},
"binds": {
"route": "${dir:state}/route.json"
"route": "/var/lib/novox.be/route.json"
},
"listens": [
{
"name": "web",
"port": 8080,
"port": 4000,
"protocol": "tcp",
"from": "mesh",
"why": "the public website over http; its public name is a route grant, and route-proxy reaches it on this published port"
"why": "the public website over http; the public name novox.be is a route grant, and route-proxy reaches it on this published port"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/novox.be",
"mode": "0700"
},
{
"id": "net",
"type": "network",
"name": "website"
"name": "novox-be"
},
{
"id": "server",
"type": "container",
"name": "website",
"name": "novox-be",
"image": "registry-api.novox.be/novox/www@sha256:aa7ed20a293e1d7444c5c5d59b6d8bdb382bad159559a22e006810e379cae69c",
"network": "website",
"network": "novox-be",
"ports": [
"8080"
],
"names-on-purpose": {
"registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)"
}
]
}
]
}
+9 -10
View File
@@ -10,8 +10,8 @@
],
"consumes": [],
"own-secrets": {
"broker": "${dir:mesh-state}/broker",
"password": "${dir:mesh-state}/password"
"broker": "/var/lib/mesh/nzbget/broker",
"password": "/var/lib/mesh/nzbget/password"
},
"listens": [
{
@@ -24,7 +24,6 @@
],
"accesses": [
{
"id": "downloads",
"path": "/services/media/downloads",
"mode": "read-write"
}
@@ -33,8 +32,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/nzbget",
"mode": "0700"
},
{
"id": "config",
@@ -58,13 +57,13 @@
],
"volumes": [
"/services/nzbget/config:/config",
"${access:downloads}:/downloads"
"/services/media/downloads:/downloads"
]
},
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/nzbget/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -75,9 +74,9 @@
"name": "mesh-nzbget",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/password:/run/secrets/password:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"/var/lib/mesh/nzbget/broker:/run/secrets/broker:ro",
"/var/lib/mesh/nzbget/password:/run/secrets/password:ro",
"/var/lib/mesh/nzbget/config.json:/run/config/config.json:ro",
"/services/nzbget/config:/var/lib/nzbget/config:ro"
],
"env": {
+9 -9
View File
@@ -9,8 +9,8 @@
"request.approved"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker",
"api-key": "${dir:mesh-state}/api-key"
"broker": "/var/lib/mesh/ombi/broker",
"api-key": "/var/lib/mesh/ombi/api-key"
},
"listens": [
{
@@ -25,8 +25,8 @@
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
"path": "/var/lib/mesh/ombi",
"mode": "0700"
},
{
"id": "config",
@@ -55,7 +55,7 @@
{
"id": "runtime-config",
"type": "file",
"path": "${dir:mesh-state}/config.json",
"path": "/var/lib/mesh/ombi/config.json",
"mode": "0600",
"content": "{}\n",
"merge": "json"
@@ -66,9 +66,9 @@
"name": "mesh-ombi",
"network": "host",
"volumes": [
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:mesh-state}/api-key:/run/secrets/api-key:ro",
"${dir:mesh-state}/config.json:/run/config/config.json:ro",
"/var/lib/mesh/ombi/broker:/run/secrets/broker:ro",
"/var/lib/mesh/ombi/api-key:/run/secrets/api-key:ro",
"/var/lib/mesh/ombi/config.json:/run/config/config.json:ro",
"/services/ombi/config:/var/lib/ombi/config:ro"
],
"env": {
@@ -94,7 +94,7 @@
}
},
"binds": {
"route": "${dir:mesh-state}/route.json"
"route": "/var/lib/mesh/ombi/route.json"
},
"build": {
"on": [
+3 -3
View File
@@ -23,10 +23,10 @@
"listens": [
{
"name": "web",
"port": 9070,
"port": 80,
"protocol": "tcp",
"from": "mesh",
"why": "the document server over http; its public name is a route grant, and route-proxy reaches it on this published port"
"why": "the document server over http; the public name office.novox.be is a route grant, and route-proxy reaches it on this published port"
}
],
"resources": [
@@ -93,7 +93,7 @@
"${dir:state}/server.env"
],
"ports": [
"9070:80"
"80"
],
"volumes": [
"${dir:logs}:/var/log/onlyoffice",
+6 -5
View File
@@ -9,21 +9,22 @@
"model-access"
],
"binds": {
"model-access": "${dir:state}/model.json"
"model-access": "/var/lib/openai-consumer/model.json"
},
"secrets": {
"model-access": "${dir:state}/api-key"
"model-access": "/var/lib/openai-consumer/api-key"
},
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/openai-consumer",
"mode": "0700"
},
{
"id": "config",
"type": "directory",
"path": "/var/lib/openai-consumer/config",
"mode": "0700"
},
{
@@ -37,7 +38,7 @@
"/app/modules/openai-consumer/dist/apply/index.js"
],
"volumes": [
"${dir:state}:/run/state"
"/var/lib/openai-consumer:/run/state"
],
"env": {
"MESH_MODEL_ACCESS_SECRET_FILE": "/run/state/api-key",
+5 -8
View File
@@ -15,7 +15,7 @@
}
},
"binds": {
"route": "${dir:state}/route.json"
"route": "/var/lib/photos-eef/route.json"
},
"listens": [
{
@@ -23,15 +23,15 @@
"port": 4012,
"protocol": "tcp",
"from": "mesh",
"why": "the eef photos client site over http; its public name is a route grant, and route-proxy reaches it on this published port"
"why": "the eef photos client site over http; the public name eef.novox.be is a route grant, and route-proxy reaches it on this published port"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/photos-eef",
"mode": "0700"
},
{
"id": "net",
@@ -46,10 +46,7 @@
"network": "photos-eef",
"ports": [
"80"
],
"names-on-purpose": {
"registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)"
}
]
}
]
}
+5 -8
View File
@@ -15,7 +15,7 @@
}
},
"binds": {
"route": "${dir:state}/route.json"
"route": "/var/lib/photos-filip/route.json"
},
"listens": [
{
@@ -23,15 +23,15 @@
"port": 4013,
"protocol": "tcp",
"from": "mesh",
"why": "the filip photos client site over http; its public name is a route grant, and route-proxy reaches it on this published port"
"why": "the filip photos client site over http; the public name filip.novox.be is a route grant, and route-proxy reaches it on this published port"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
"path": "/var/lib/photos-filip",
"mode": "0700"
},
{
"id": "net",
@@ -46,10 +46,7 @@
"network": "photos-filip",
"ports": [
"80"
],
"names-on-purpose": {
"registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)"
}
]
}
]
}

Some files were not shown because too many files have changed in this diff Show More