Convert four hal modules: lidarr, mongodb, mssql, mosquitto
Mirrors the proven catalog patterns field-for-field: - lidarr -> the Servarr twin of radarr/sonarr (API v1, artist content); no provisioner (it is a consumer app). - mongodb -> postgres shape: mongodb-database provider, provisioner mints a per-consumer db+user (ADR 0053), client shells to mongosh (no npm driver, the psql convention). - mssql -> postgres shape: mssql-database provider, sqlcmd client. - mosquitto -> redis shape: mqtt-topic provider via the Dynamic Security plugin, deliberately avoiding hal's password_file (that file is nox issue 011 exactly); provisioner mints a per-consumer MQTT client+role. All four typecheck (strict, NodeNext) against the built @novox/mesh-sdk, and their service images are digest-pinned to resolved registry digests. The mesh-runtime-<mod> images keep the all-zeros placeholder the pipeline pins, as postgres/redis do, and must bundle each module's CLI (mongosh/sqlcmd/ mosquitto_ctrl) as mesh-runtime-postgres bundles psql. Not yet lab-verified: each module lists in-code what an integration test must prove (auth model, provisioner reconcile, mosquitto dynsec bootstrap ordering). Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
// mosquitto's admin client — mosquitto's own code, living in the module (novox/hq ADR 0044). Both
|
||||
// this module's tools and its provisioner import it, and nothing outside mosquitto does.
|
||||
//
|
||||
// Client, role and ACL administration is driven through `mosquitto_ctrl dynsec`, not a hand-rolled
|
||||
// MQTT stack: the module may take NO npm dependency beyond @novox/mesh-sdk, and mosquitto ships the
|
||||
// exact admin client for its Dynamic Security plugin — so it shells out to it, the same way postgres
|
||||
// drives itself through `psql`, minio through `mc` and mailu through `doveadm`. One boundary,
|
||||
// `ctl()`, and every method is built on it.
|
||||
//
|
||||
// Why the Dynamic Security plugin and not `password_file`: dynsec creates and revokes clients while
|
||||
// the broker runs, over an admin connection, with no broker restart and no file the host rewrites —
|
||||
// the true analog of redis's runtime ACL users. A `password_file` would have to be re-read on a
|
||||
// SIGHUP the runtime cannot cleanly send across containers, and — declared as a managed file — would
|
||||
// be rewritten by the host on every reconcile, wiping every provisioned user (novox/nox issue 011).
|
||||
// The one cost dynsec carries is the bootstrap file; see initBootstrapFile() and the module README.
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
export interface MqttConn {
|
||||
readonly host: string;
|
||||
readonly port: number;
|
||||
/** The Dynamic Security admin client the runtime authenticates as. */
|
||||
readonly adminUser: string;
|
||||
readonly adminPassword: string;
|
||||
}
|
||||
|
||||
export class MosquittoClient {
|
||||
constructor(private readonly conn: MqttConn) {}
|
||||
|
||||
/**
|
||||
* Build from the module's resolved environment. Reads MESH_MQTT_* first (the documented names),
|
||||
* falling back to the MESH_PROVISION_* keys the manifest already sets on the provisioner
|
||||
* container. Throws if it cannot find a host and an admin password — the right failure, because
|
||||
* without them nothing it does can work.
|
||||
*/
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): MosquittoClient {
|
||||
const endpoint = env.MESH_PROVISION_MQTT ?? ""; // "host:port"
|
||||
const host = env.MESH_MQTT_HOST ?? (endpoint ? endpoint.split(":")[0] : undefined);
|
||||
const port =
|
||||
Number(env.MESH_MQTT_PORT ?? (endpoint.includes(":") ? endpoint.split(":")[1] : "") ?? "1883") || 1883;
|
||||
const adminUser = env.MESH_MQTT_ADMIN_USER ?? env.MESH_PROVISION_ADMIN_USER ?? "mesh-admin";
|
||||
const adminPassword = env.MESH_MQTT_PASSWORD ?? readSecretFile(env.MESH_PROVISION_PASSWORD_FILE);
|
||||
if (!host || !adminPassword) {
|
||||
throw new Error(
|
||||
"mosquitto host or admin password is not set — mosquitto's own code cannot reach the broker",
|
||||
);
|
||||
}
|
||||
return new MosquittoClient({ host, port, adminUser, adminPassword: adminPassword ?? "" });
|
||||
}
|
||||
|
||||
get host(): string {
|
||||
return this.conn.host;
|
||||
}
|
||||
|
||||
get port(): number {
|
||||
return this.conn.port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one `mosquitto_ctrl dynsec <args>` command against the broker as the admin client and return
|
||||
* its stdout. Connects over MQTT with the verified connect flags `-h`/`-p`/`-u`/`-P`. A non-zero
|
||||
* exit rejects — a failed command is an error here, not a success with a warning.
|
||||
*
|
||||
* The admin password rides on argv (`-P`): mosquitto_ctrl 2.x exposes no password env var and no
|
||||
* password file for a broker connection — its only non-interactive mechanism is `-P`, its only
|
||||
* other mechanism an interactive prompt. This is a real mosquitto limitation, not a choice; unlike
|
||||
* psql's PGPASSWORD there is nothing cleaner to reach for. The exposure is momentary and confined
|
||||
* to this single-purpose runtime container; see the module README.
|
||||
*/
|
||||
async ctl(...args: string[]): Promise<string> {
|
||||
const base = [
|
||||
"-h", this.conn.host,
|
||||
"-p", String(this.conn.port),
|
||||
"-u", this.conn.adminUser,
|
||||
"-P", this.conn.adminPassword,
|
||||
];
|
||||
const { stdout } = await run("mosquitto_ctrl", [...base, "dynsec", ...args], {
|
||||
maxBuffer: 16 << 20,
|
||||
});
|
||||
return stdout;
|
||||
}
|
||||
|
||||
/** Whether a dynsec client with this username already exists. */
|
||||
async clientExists(username: string): Promise<boolean> {
|
||||
try {
|
||||
await this.ctl("getClient", username);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create (or reset to a known state) a client scoped to one topic namespace, idempotently. The
|
||||
* client is confined to `<prefix>/#` by a same-named role: it may publish to, subscribe to and
|
||||
* receive on exactly its own subtree and nothing else — the MQTT analog of redis's keyspace-scoped
|
||||
* ACL user. Called again for an existing client, it resets the password and re-asserts the ACLs.
|
||||
*/
|
||||
async createScopedClient(username: string, password: string, topicPrefix: string): Promise<void> {
|
||||
const role = username; // one role per client, named for it
|
||||
const pattern = `${topicPrefix}/#`;
|
||||
|
||||
if (await this.clientExists(username)) {
|
||||
await this.ctl("setClientPassword", username, password);
|
||||
} else {
|
||||
await this.ctl("createClient", username, "-p", password);
|
||||
}
|
||||
|
||||
// A role carrying exactly this client's topic ACLs. createRole fails if it already exists; that
|
||||
// is fine — the setRoleACL calls below assert the intended state either way.
|
||||
await ignoreExisting(this.ctl("createRole", role));
|
||||
for (const acl of ["publishClientSend", "publishClientReceive", "subscribePattern"]) {
|
||||
// allow (1) this client to send to, receive on, and subscribe under its own subtree.
|
||||
await this.ctl("addRoleACL", role, acl, pattern, "allow");
|
||||
}
|
||||
await ignoreExisting(this.ctl("addClientRole", username, role));
|
||||
}
|
||||
|
||||
/** Remove a client and the per-client role created for it, idempotently. */
|
||||
async deleteScopedClient(username: string): Promise<void> {
|
||||
await ignoreMissing(this.ctl("deleteClient", username));
|
||||
await ignoreMissing(this.ctl("deleteRole", username));
|
||||
}
|
||||
|
||||
/** The dynsec client list, parsed from `listClients`. */
|
||||
async listClients(): Promise<string[]> {
|
||||
const out = await this.ctl("listClients");
|
||||
return out
|
||||
.split(/\r?\n/)
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the Dynamic Security bootstrap file offline, creating the admin client the plugin loads at
|
||||
* broker startup. This is a one-time seed, NOT part of the reconcile loop: run once before the
|
||||
* broker first starts, against the same path the broker's `plugin_opt_config_file` names. It must
|
||||
* never be a host-reconciled managed file — see the module README and novox/nox issue 011.
|
||||
*/
|
||||
async initBootstrapFile(configFile: string): Promise<void> {
|
||||
// `dynsec init <file> <admin-username> [admin-password]` is an offline file operation — it does
|
||||
// not connect to the broker. The password is a positional argument (omitting it prompts).
|
||||
await run("mosquitto_ctrl", ["dynsec", "init", configFile, this.conn.adminUser, this.conn.adminPassword], {
|
||||
maxBuffer: 16 << 20,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Generate a URL-safe password with no argv- or MQTT-hostile characters. */
|
||||
export function generatePassword(): string {
|
||||
return randomBytes(24).toString("base64url");
|
||||
}
|
||||
|
||||
/** Swallow a "already exists" failure so create paths are idempotent; rethrow anything else. */
|
||||
async function ignoreExisting(p: Promise<string>): Promise<void> {
|
||||
try {
|
||||
await p;
|
||||
} catch (err) {
|
||||
if (!/exist/i.test(String(err))) throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Swallow a "not found" failure so delete paths are idempotent; rethrow anything else. */
|
||||
async function ignoreMissing(p: Promise<string>): Promise<void> {
|
||||
try {
|
||||
await p;
|
||||
} catch (err) {
|
||||
if (!/not\s*found|does not exist|no such/i.test(String(err))) throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function readSecretFile(path: string | undefined): string | undefined {
|
||||
if (!path) return undefined;
|
||||
try {
|
||||
return readFileSync(path, "utf8").trim();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// mosquitto's events entrypoint, loaded by the per-node tool host (the provisioner container runs
|
||||
// ./provisioner separately). The topic lifecycle events are EMITTED from the provisioner, where the
|
||||
// lifecycle actually happens (novox/hq ADR 0046/0047):
|
||||
// module.mosquitto.topic.provisioned — a consumer's client + scoped role was created
|
||||
// module.mosquitto.topic.deprovisioned — that client was removed
|
||||
// Here in the tool host we react to them, keeping a lightweight audit trail of who was granted a
|
||||
// topic namespace and who lost one — observability the provider itself is best placed to log.
|
||||
|
||||
import { on } from "@novox/mesh-sdk/events";
|
||||
|
||||
interface TopicEvent {
|
||||
consumer: string;
|
||||
username: string;
|
||||
topicPrefix?: string;
|
||||
}
|
||||
|
||||
await on<TopicEvent>("module.mosquitto.topic.provisioned", async (e) => {
|
||||
console.log(`[mosquitto] topic provisioned for ${e.body.consumer} (client ${e.body.username})`);
|
||||
});
|
||||
|
||||
await on<TopicEvent>("module.mosquitto.topic.deprovisioned", async (e) => {
|
||||
console.log(`[mosquitto] topic deprovisioned for ${e.body.consumer} (client ${e.body.username})`);
|
||||
});
|
||||
|
||||
console.log("[mosquitto] auditing topic lifecycle events");
|
||||
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"module": "mosquitto",
|
||||
"version": "1",
|
||||
"provides": [
|
||||
{
|
||||
"name": "mqtt-topic",
|
||||
"scope": "mesh"
|
||||
}
|
||||
],
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"emits": [
|
||||
"module.mosquitto.topic.provisioned",
|
||||
"module.mosquitto.topic.deprovisioned"
|
||||
],
|
||||
"consumes": [
|
||||
"module.mosquitto.topic.provisioned",
|
||||
"module.mosquitto.topic.deprovisioned"
|
||||
],
|
||||
"serves": {
|
||||
"mqtt-topic": {}
|
||||
},
|
||||
"receives": {
|
||||
"mqtt-topic": "/var/lib/mosquitto-module/grants/mesh.json"
|
||||
},
|
||||
"grants": {
|
||||
"mqtt-topic": "/var/lib/mosquitto-module/grants"
|
||||
},
|
||||
"own-secrets": {
|
||||
"admin": "/var/lib/mosquitto-module/admin.secret",
|
||||
"broker": "/var/lib/mesh/mosquitto/broker"
|
||||
},
|
||||
"listens": [
|
||||
{
|
||||
"port": 1883,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"why": "modules on any machine that were granted a topic namespace"
|
||||
},
|
||||
{
|
||||
"port": 8081,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"why": "the same broker over MQTT-on-WebSockets, for browser clients"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "mesh-state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh/mosquitto",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mosquitto-module",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "grants-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mosquitto-module/grants",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "data",
|
||||
"type": "directory",
|
||||
"path": "/services/mosquitto/data",
|
||||
"mode": "0700",
|
||||
"owner": "1883:1883"
|
||||
},
|
||||
{
|
||||
"id": "server-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mosquitto-module/mosquitto.conf",
|
||||
"mode": "0600",
|
||||
"owner": "1883:1883",
|
||||
"content": "persistence true\npersistence_location /mosquitto/data\n\nlog_dest stdout\nlog_type warning\nlog_type error\nlog_type notice\n\n# Every client authenticates; identities and their per-topic ACLs are managed\n# at runtime by the dynamic security plugin, whose store the plugin itself owns.\nallow_anonymous false\nplugin /usr/lib/mosquitto_dynamic_security.so\nplugin_opt_config_file /mosquitto/data/dynamic-security.json\n\n# MQTT listener\nlistener 1883\n\n# MQTT-over-WebSockets listener\nlistener 8081\nprotocol websockets\n"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
"type": "network",
|
||||
"name": "mosquitto"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
"name": "mosquitto",
|
||||
"image": "eclipse-mosquitto@sha256:6f8d8a947c506f8a2290ec65cd4bd2bc7cb4d43fb5f6271f861cb013e2ef9797",
|
||||
"network": "mosquitto",
|
||||
"ports": [
|
||||
"1883",
|
||||
"8081"
|
||||
],
|
||||
"volumes": [
|
||||
"/services/mosquitto/data:/mosquitto/data",
|
||||
"/var/lib/mosquitto-module/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "runtime",
|
||||
"type": "container",
|
||||
"name": "mesh-mosquitto",
|
||||
"image": "mesh-runtime-mosquitto@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"network": "mosquitto",
|
||||
"volumes": [
|
||||
"/var/lib/mesh/mosquitto/broker:/run/secrets/broker:ro",
|
||||
"/var/lib/mosquitto-module/grants:/var/lib/mosquitto-module/grants:ro",
|
||||
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "/run/secrets/broker",
|
||||
"MESH_RECEIVES": "/var/lib/mosquitto-module/grants/mesh.json",
|
||||
"MESH_PROVISION_MQTT": "mosquitto:1883",
|
||||
"MESH_PROVISION_ADMIN_USER": "mesh-admin",
|
||||
"MESH_PROVISION_PASSWORD_FILE": "/run/secrets/admin"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "@novox/module-mosquitto",
|
||||
"version": "0.1.0",
|
||||
"description": "mosquitto — provides the mesh mqtt-topic interface. Its admin client, provisioner, tools and events live here (novox/hq ADR 0044).",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.6.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// mosquitto's provisioner — the adapter that makes mosquitto a provider of the mesh `mqtt-topic`
|
||||
// interface. The reconcile loop, the contributions file, and reading the mesh's minted password are
|
||||
// the sdk harness's; this writes only the per-service half: how mosquitto creates and removes a
|
||||
// per-consumer MQTT client (novox/hq ADR 0044/0045/0053).
|
||||
//
|
||||
// The `mqtt-topic` interface: a consumer connects as `as` with the password the mesh minted, and
|
||||
// publishes and subscribes under `<as>/#`, isolated from every other consumer by a Dynamic Security
|
||||
// role scoped to exactly that subtree.
|
||||
//
|
||||
// **The login and password are the mesh's, not the provisioner's (ADR 0053).** The mesh derives the
|
||||
// login and hands it to both ends so they agree, and mints the password and delivers a copy to each.
|
||||
// mosquitto creates exactly that client with exactly that password — a name or password the
|
||||
// provisioner invented is one the consumer could never present.
|
||||
|
||||
import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner";
|
||||
import { emit } from "@novox/mesh-sdk/events";
|
||||
import { MosquittoClient } from "../client.js";
|
||||
|
||||
const mosquitto = MosquittoClient.fromEnv();
|
||||
|
||||
/** Emit a lifecycle event without letting a broker hiccup fail the provisioning itself. */
|
||||
async function announce(type: string, body: Record<string, string>): Promise<void> {
|
||||
try {
|
||||
await emit(type, body);
|
||||
} catch (err) {
|
||||
console.error(`[provisioner:mqtt-topic] emit ${type} failed: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
runProvisioner("mqtt-topic", {
|
||||
async create(p: Provision): Promise<void> {
|
||||
// The topic subtree is scoped to the consumer's own login, so one cannot read another's topics.
|
||||
const topicPrefix = p.as;
|
||||
await mosquitto.createScopedClient(p.as, p.password, topicPrefix);
|
||||
await announce("module.mosquitto.topic.provisioned", {
|
||||
consumer: p.consumer ?? "",
|
||||
username: p.as,
|
||||
topicPrefix,
|
||||
});
|
||||
},
|
||||
|
||||
async remove(p: { as: string }): Promise<void> {
|
||||
await mosquitto.deleteScopedClient(p.as);
|
||||
await announce("module.mosquitto.topic.deprovisioned", { username: p.as });
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
// mosquitto's tools — mosquitto's own code (novox/hq ADR 0044), importing mosquitto's own admin
|
||||
// client. They return structured data; the mesh serves them through the sdk's tool harness.
|
||||
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { MosquittoClient } from "../client.js";
|
||||
|
||||
export function getMosquittoTools(mosquitto: MosquittoClient): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "mqtt_list_clients",
|
||||
description: "List the Dynamic Security clients registered on the mosquitto broker.",
|
||||
input: {},
|
||||
run: async () => ({ clients: await mosquitto.listClients() }),
|
||||
},
|
||||
{
|
||||
name: "mqtt_get_client",
|
||||
description: "Show one Dynamic Security client — its roles and enabled state.",
|
||||
input: { username: { type: "string", description: "the client's username" } },
|
||||
run: async (args) => {
|
||||
const username = String(args.username ?? "");
|
||||
if (!username) throw new Error("mqtt_get_client: username is required");
|
||||
return { username, detail: await mosquitto.ctl("getClient", username) };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mqtt_ctrl",
|
||||
description:
|
||||
"Run an arbitrary 'mosquitto_ctrl dynsec' subcommand, e.g. 'listRoles', 'getRole myrole'. Admin surface.",
|
||||
input: { command: { type: "string", description: "the dynsec subcommand and its arguments, space-separated" } },
|
||||
run: async (args) => {
|
||||
const parts = tokenize(String(args.command ?? ""));
|
||||
if (parts.length === 0) throw new Error("mqtt_ctrl: empty command");
|
||||
const output = await mosquitto.ctl(...parts);
|
||||
return { command: parts.join(" "), output };
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/** Split a command line into arguments, honouring double-quoted spans. */
|
||||
function tokenize(command: string): string[] {
|
||||
const matches = command.match(/(?:[^\s"]+|"[^"]*")+/g) ?? [];
|
||||
return matches.map((p) => p.replace(/^"|"$/g, ""));
|
||||
}
|
||||
|
||||
// The tools exist only when the broker can be reached from the environment; without it, mosquitto
|
||||
// contributes none rather than failing the whole tool runtime.
|
||||
registerModuleTools("mosquitto", (env) => {
|
||||
try {
|
||||
return getMosquittoTools(MosquittoClient.fromEnv(env));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["client.ts", "index.ts", "provisioner/index.ts", "tools/index.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user