mesh-catalog/modules/mosquitto/bootstrap/index.ts

// mosquitto's run-once bootstrap — mosquitto's own code (novox/hq ADR 0039), run once before the
// broker first starts (ADR 0052). The Dynamic Security plugin refuses to bring the broker up unless
// `dynamic-security.json` already holds an admin client, and nothing in a reconcile loop ever seeds
// that file — a state declaration describes what should exist, not a step that runs. This is that
// step: it writes the seed offline, exactly once, and exits. The host runs it to completion and only
// then starts the broker container the manifest places after it.
//
// It runs in the module's own runtime image, under the module's own account, as `mesh-tools run`
// imports it — no broker connection, because seeding is an offline file operation and there is no
// broker to reach yet. `mosquitto_ctrl dynsec init` writes the file; nothing here talks to a server.
//
// **Two disciplines make the seed safe to live beside a file the plugin then grows:**
//
//   - It writes only when the file is absent, and never reconciles it. Once the broker is up, the
//     dynsec plugin owns that file and rewrites it on every client it creates; re-seeding would wipe
//     every provisioned client (novox/hq 04-ISSUES/035). The host's completion marker keeps the step
//     from re-running; this absent-check keeps the one pass it does run from clobbering.
//   - It hands the file to the broker's user. The broker runs as uid 1883 and must both READ the
//     seed at startup and PERSIST to it as clients come and go; this container runs as root and would
//     otherwise leave a root-owned file the broker at 1883 can neither read (if 0600) nor rewrite.
//     So after writing, it chowns the file to 1883:1883 and sets 0600 — the broker's to read and to
//     grow, and no one else's. This is the ownership question ADR 0052 left for the lab to settle.
//
// **And it keeps the broker's admin password the mesh's** (novox/hq issue 282). The admin password is
// an own secret the mesh may rotate (ADR 0228): it makes a new value, writes it, and this step runs
// again (its `restart-on` names the secret). The store holds the password's hash, which only the
// broker changes, and only for an admin who presents the password it holds — so this step keeps the
// value it last applied, in a 0600 file of the module's own state, and when the mesh's value is no
// longer that one it connects with the applied one and sets the new one, online, on stdin. The
// broker keeps running and loses no client; the step then records the new value as applied. Until
// it has, nothing of this module can administer the broker, so a re-key that cannot be done fails
// the step, loudly, by name and never by value.

import { chownSync, chmodSync, existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { MosquittoClient } from "../client.js";

// The broker (eclipse-mosquitto) runs as this uid/gid; the seeded store must be its to read and
// rewrite. Overridable for a broker image that runs as a different user.
const BROKER_UID = Number(process.env.MESH_MQTT_BROKER_UID ?? "1883") || 1883;
const BROKER_GID = Number(process.env.MESH_MQTT_BROKER_GID ?? "1883") || 1883;

// The same path the broker's `plugin_opt_config_file` names, reached through the shared data volume.
const configFile = process.env.MESH_DYNSEC_FILE ?? "/mosquitto/data/dynamic-security.json";

// Where this step keeps the admin password it last applied to the store (see the header).
const appliedFile = process.env.MESH_MQTT_ADMIN_APPLIED_FILE ?? "";
// The broker's container, entered to re-key the admin and started if a re-key finds it stopped.
const container = process.env.MESH_MQTT_CTRL_CONTAINER ?? "";

const mosquitto = MosquittoClient.fromEnv();
const wanted = readSecret(process.env.MESH_PROVISION_PASSWORD_FILE);
if (!wanted) throw new Error("the broker's admin password is not readable; nothing was seeded or re-keyed");

if (existsSync(configFile)) {
  // Already seeded — and possibly grown by the running plugin since. Never rewritten here.
  console.log(`[mosquitto:bootstrap] ${configFile} already exists; leaving it untouched`);
  await keepAdminPassword();
} else {
  await mosquitto.initBootstrapFile(configFile);
  // Hand the store to the broker's user so it can read the seed and persist to it (see the header).
  chownSync(configFile, BROKER_UID, BROKER_GID);
  chmodSync(configFile, 0o600);
  recordApplied(wanted);
  console.log(
    `[mosquitto:bootstrap] seeded ${configFile} with the dynsec admin client, owned by ${BROKER_UID}:${BROKER_GID}`,
  );
}

/**
 * Make the store's admin password the mesh's. Nothing to do when the value applied last is the
 * mesh's; otherwise the broker is asked to take the new one from an admin holding the old.
 */
async function keepAdminPassword(): Promise<void> {
  if (!appliedFile) {
    console.log("[mosquitto:bootstrap] no applied-password file is named; the admin password is not kept here");
    return;
  }
  const applied = readSecret(appliedFile);
  if (applied === wanted) return;

  let now = await mosquitto.adminAccepted(wanted);
  if (now === undefined && applied) now = await startAndAsk(wanted);
  if (now === true) {
    // The broker already takes it: a re-key that ran before its record was written, or the first run
    // of this step on a store seeded before it kept a record.
    recordApplied(wanted);
    console.log("[mosquitto:bootstrap] the broker takes the mesh's admin password; recorded as applied");
    return;
  }
  if (now === undefined && applied) {
    // Started, or not startable, and still not answering: the store keeps the old password until the
    // broker takes the new one, so this step fails rather than records what did not happen.
    throw new Error(`the mesh's admin password changed and the broker cannot be reached to take it: start ` +
      `${container || "the broker"} and apply again (novox/hq issue 282)`);
  }
  if (now === undefined) {
    // Nothing applied is known and the broker is not up: the store was seeded with the value it holds
    // now as far as anything here can tell, and the first connection says otherwise if it was not.
    recordApplied(wanted);
    console.log("[mosquitto:bootstrap] the broker is not running; the mesh's admin password is recorded as applied, unverified");
    return;
  }
  if (!applied) {
    throw new Error("the broker refuses the mesh's admin password, and no password applied before is recorded " +
      "to change it with: the store's admin client was given another one (novox/hq issue 282)");
  }
  const old = mosquitto.withAdminPassword(applied);
  if ((await old.adminAccepted(applied)) !== true) {
    throw new Error("the broker refuses both the mesh's admin password and the one applied before it; " +
      "the admin client cannot be re-keyed from here (novox/hq issue 282)");
  }
  await old.ctlWithPassword(wanted, "setClientPassword", mosquitto.adminUser);
  if ((await mosquitto.adminAccepted(wanted)) !== true) {
    throw new Error("the broker was asked to take the mesh's new admin password and still refuses it");
  }
  recordApplied(wanted);
  console.log("[mosquitto:bootstrap] the broker's admin password was replaced by the mesh's new one, online");
}

/** Start the broker's container when it is stopped, and ask again once it answers. */
async function startAndAsk(password: string): Promise<boolean | undefined> {
  if (!container) return undefined;
  try {
    await promisify(execFile)("docker", ["start", container]);
  } catch {
    return undefined; // no such container yet: the apply creates it after this step
  }
  for (let i = 0; i < 20; i++) {
    const answer = await mosquitto.adminAccepted(password);
    if (answer !== undefined) return answer;
    await new Promise((r) => setTimeout(r, 1000));
  }
  return undefined;
}

/** Record the value now applied: written whole, 0600, then moved into place. */
function recordApplied(value: string): void {
  if (!appliedFile) return;
  const next = `${appliedFile}.next`;
  writeFileSync(next, value, { mode: 0o600 });
  chmodSync(next, 0o600);
  renameSync(next, appliedFile);
}

function readSecret(path: string | undefined): string {
  if (!path) return "";
  try {
    return readFileSync(path, "utf8").trim();
  } catch {
    return "";
  }
}