mesh-lab/src/lifecycle/place.ts

/**
 * `place:` — putting something inside the machines.
 *
 * Until now the lab raised an underlay and put nothing on it: correct, and useless, because
 * the thing it exists to test did not exist (novox/hq 03-DESIGN/00-as-is/11-the-lab.md). Tier
 * 0 now does, so this is the seam where the lab acquires a consumer.
 *
 * Only `host` is placeable. Everything else in the placement vocabulary — the foundation, a
 * control plane, a forge — is still refused by name rather than ignored, because a scenario
 * that declares something and raises without it is the fault this lab was built to catch
 * (novox/hq 04-ISSUES/003).
 */

import type { Scenario } from "../declaration/types.ts";
import { spawn } from "node:child_process";
import { unlink } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { incus, incusOk, succeeds } from "../incus/client.ts";
import { around, log, shorten } from "../log.ts";
import { mustBeHandedOver, repositoryOf, type HeldImage } from "../pinning.ts";

/**
 * What this stage can put inside a machine.
 *
 * `image:<reference>` is the one that takes an argument — `image:alpine@sha256:...` — because
 * WHICH image is the point of placing one.
 */
export const PLACEABLE = ["host", "runtime"] as const;
export type Placeable = (typeof PLACEABLE)[number];

/** The prefix that marks a container image, and what follows it is passed through unchanged. */
export const IMAGE_PREFIX = "image:";

/** Whether an artifact names something this stage can place. */
export function isPlaceable(artifact: string): boolean {
  return (PLACEABLE as readonly string[]).includes(artifact) ||
    (artifact.startsWith(IMAGE_PREFIX) && artifact.slice(IMAGE_PREFIX.length).trim() !== "");
}

/** Where the host binary lives on a machine once placed. */
export const HOST_PATH = "/usr/local/bin/mesh-host";

/**
 * Where the installer lives on a machine once placed.
 *
 * **Beside the host, because it is the same tier and the same delivery** (novox/hq ADR 0067):
 * bootstrapping is done by hand and it changes a machine, which is what tier 0 is — but `mesh-host`
 * says of itself that it connects to nothing and listens on nothing, and an installer that loads
 * images and interrogates a control plane cannot be folded into it without making that sentence
 * false. Two programs, one shelf.
 */
export const BOOTSTRAP_PATH = "/usr/local/bin/mesh-bootstrap";

export interface Placement {
  machine: string;
  artifacts: string[];
}

/**
 * Resolve `place:` to one list per machine.
 *
 * `all:` applies to every machine; a per-machine entry **overrides** it rather than adding to
 * it, which is what the declaration design says and is worth being exact about — a scenario
 * naming one artifact for one machine gets that artifact and not that artifact plus the rest.
 */
export function planPlacements(scenario: Scenario): Placement[] {
  const place = scenario.place;
  if (!place) return [];

  const all = place.all ?? [];
  return Object.keys(scenario.machines).map((machine) => {
    const own = place[machine];
    return { machine, artifacts: own ?? all };
  }).filter((p) => p.artifacts.length > 0);
}

/**
 * The host binary to place, from the environment.
 *
 * Deliberately an explicit path rather than a search. The declaration design leaves *where
 * `place:` gets its artifacts from* open — before the mesh is self-hosting they come from
 * outside, afterwards from the mesh — and it suggests a named source rather than a path. This
 * is neither: it is the smallest thing that works while that stays undecided, and it refuses
 * loudly rather than guessing, so nothing here hardens into the answer by accident.
 */
export function hostBinaryPath(): string | null {
  return process.env["MESH_LAB_HOST_BINARY"] ?? null;
}

/**
 * The installer to place, from the environment. Same rule as the host binary: an explicit path,
 * and nothing is guessed.
 */
export function bootstrapBinaryPath(): string | null {
  return process.env["MESH_LAB_BOOTSTRAP_BINARY"] ?? null;
}

export class PlacementError extends Error {
  readonly machine: string;

  constructor(machine: string, message: string) {
    super(message);
    this.name = "PlacementError";
    this.machine = machine;
  }
}

export interface PlacedHost {
  machine: string;
  /** What the host reported about the machine, read back from it. */
  profile: unknown;
  version: string;
}

/**
 * Put the host on a machine and ask it what the machine is.
 *
 * The result is read back from the running binary, never assumed from the fact that the copy
 * succeeded (novox/hq ADR 0018). A file arriving is not a host working, which is the same
 * distinction the host itself makes about installed packages.
 */
export async function placeHost(
  instanceName: string,
  machine: string,
  binary: string,
  log: (message: string) => void = () => {},
): Promise<PlacedHost> {
  await incus(["file", "push", binary, `${instanceName}${HOST_PATH}`, "--mode", "0755"], 180_000);

  // Read back that it is there and executable before trusting it to answer questions.
  const version = (await incusOk(["exec", instanceName, "--", HOST_PATH, "version"], 60_000))?.trim();
  if (!version) {
    throw new PlacementError(
      machine,
      `the host binary was copied to ${machine} but does not run there. A file arriving is ` +
        `not a host working.`,
    );
  }

  const reported = await incusOk(
    ["exec", instanceName, "--", HOST_PATH, "profile", "--json"],
    120_000,
  );
  if (!reported) {
    throw new PlacementError(
      machine,
      `the host runs on ${machine} (${version}) but reported no profile. A host that cannot ` +
        `say what a machine is cannot be asked to change it.`,
    );
  }

  let profile: unknown;
  try {
    profile = JSON.parse(reported);
  } catch (err) {
    throw new PlacementError(
      machine,
      `the host on ${machine} reported something that is not a profile: ${(err as Error).message}`,
    );
  }

  log(`  placed the host on ${machine} (${version})`);
  return { machine, profile, version };
}

/**
 * Put the installer on a machine and ask it what it is.
 *
 * The same shape as {@link placeHost} and for the same reason: the version is read back from the
 * running binary, because a file arriving is not a program working (novox/hq ADR 0018). The
 * installer is the larger of the two by an order of magnitude — it carries a saved container image
 * — so a copy that half-arrived is a real possibility rather than a theoretical one.
 *
 * It is placed on ONE machine, not all of them. Only the anchor is bootstrapped; every other
 * machine joins a mesh that already exists, with the host binary and a token and nothing else.
 */
export async function placeBootstrap(
  instanceName: string,
  machine: string,
  binary: string,
  log: (message: string) => void = () => {},
): Promise<string> {
  await incus(["file", "push", binary, `${instanceName}${BOOTSTRAP_PATH}`, "--mode", "0755"],
    900_000);

  const version = (await incusOk(
    ["exec", instanceName, "--", BOOTSTRAP_PATH, "version"], 60_000,
  ))?.trim();
  if (!version) {
    throw new PlacementError(
      machine,
      `the installer was copied to ${machine} and does not run there. It carries a saved ` +
        `container image and is twenty megabytes or so, which is exactly the size at which a ` +
        `truncated copy stops being theoretical.`,
    );
  }
  log(`  placed the installer on ${machine} (${version})`);
  return version;
}

/** Place everything a scenario declares. */
export async function applyPlacements(
  scenario: Scenario,
  machineNames: Map<string, string>,
  log: (message: string) => void = () => {},
): Promise<PlacedHost[]> {
  const placements = planPlacements(scenario);
  if (placements.length === 0) return [];

  // Only demanded when something actually needs it. A scenario placing a runtime and an image
  // and no host is a legitimate thing to want, and asking it for a host binary would be
  // refusing a scenario for not doing something it never said it would.
  const wantsHost = placements.some((p) => p.artifacts.includes("host"));
  const binary = hostBinaryPath();
  if (wantsHost && !binary) {
    throw new Error(
      `this scenario places the host, and no host binary was given. Set ` +
        `MESH_LAB_HOST_BINARY to a built mesh-host. Guessing at a path would place ` +
        `whatever happened to be there.`,
    );
  }

  const placed: PlacedHost[] = [];
  for (const { machine, artifacts } of placements) {
    const name = machineNames.get(machine);
    if (!name) continue;

    // Order within a machine is the order declared, because a scenario placing an image
    // before a runtime means something different from the reverse and the lab should not
    // silently reorder it into working.
    for (const artifact of artifacts) {
      if (artifact === "host") {
        placed.push(await placeHost(name, machine, binary as string, log));
      } else if (artifact === "runtime") {
        await placeRuntime(name, machine, log);
      } else if (artifact.startsWith(IMAGE_PREFIX)) {
        await placeImage(name, machine, artifact.slice(IMAGE_PREFIX.length).trim(), log);
      }
      // Anything else was refused by the validator; reaching here would be a validator bug.
    }
  }
  return placed;
}

/**
 * The base image a scenario's machines are built from, when they need a container runtime.
 *
 * A sealed scenario cannot install one: its segments use documentation ranges and there is no
 * route out (novox/hq ADR 0016). So the runtime arrives the way research 012 says everything
 * awkward should — **fetched at build time on a machine that has a network, applied on a target
 * that then needs nothing.** Building this image is that build time.
 *
 * Measured before this was written: an Arch VM installs docker in about 30 seconds, and
 * publishing the result takes about a minute and yields roughly 700 MiB. That is a one-time
 * cost paid once per lab, not per scenario.
 */
export const BASE_IMAGE_ALIAS = "mesh-lab/base";

/** How to build it, said in one place so an error can point at it. */
export const BASE_IMAGE_HOWTO =
  `Build it with: mesh-lab base build\n` +
  `  It launches a machine WITH a network, installs a container runtime, and publishes the\n` +
  `  result as '${BASE_IMAGE_ALIAS}'. A scenario is sealed and cannot do that for itself.`;

/**
 * Confirm a machine has a working container runtime.
 *
 * Asks the runtime, never the filesystem. A binary being present is not a runtime working —
 * that is 04-ISSUES/007, and it is the reason this runs `info` rather than checking a path.
 *
 * It does not INSTALL one. A sealed machine cannot fetch, so an absent runtime is a scenario
 * built on the wrong image, and saying that is more useful than failing inside a package
 * manager with no network.
 */
export async function placeRuntime(
  instanceName: string,
  machine: string,
  log: (message: string) => void = () => {},
): Promise<string> {
  const version = (await incusOk(
    ["exec", instanceName, "--", "docker", "info", "--format", "{{.ServerVersion}}"],
    60_000,
  ))?.trim();

  if (!version) {
    throw new PlacementError(
      machine,
      `${machine} has no working container runtime, and a sealed scenario cannot install one.\n` +
        `  This machine needs to be built from '${BASE_IMAGE_ALIAS}'.\n` +
        BASE_IMAGE_HOWTO,
    );
  }
  log(`  ${machine} has a container runtime (docker ${version})`);
  return version;
}

/**
 * Put a container image inside a machine.
 *
 * Exported from this workstation and loaded in the machine, because the machine cannot reach a
 * registry. The reference is passed through unchanged — including its digest — so what runs in
 * the lab is the image the declaration names rather than whatever a tag pointed at.
 */
export async function placeImage(
  instanceName: string,
  machine: string,
  reference: string,
  log: (message: string) => void = () => {},
): Promise<void> {
  // A digest reference cannot be placed this way, and finding that out here is much better
  // than finding it out when a container fails to start.
  //
  // `docker save alpine@sha256:...` produces an archive with NO repo tag — only an image ID —
  // because a repo digest exists only for an image a registry served. Loading it gives a
  // dangling image, so `docker run <that digest>` falls through to the registry, which a
  // sealed machine cannot reach. Measured, not assumed: the load says `Loaded image ID:`
  // instead of `Loaded image:`, and `docker images` then lists nothing.
  //
  // What an archive DOES keep is the image's own ID — the digest of its configuration — and that
  // is how the mesh's own images are named once loaded. See {@link loadHeldImages}.
  if (reference.includes("@sha256:")) {
    throw new PlacementError(
      machine,
      `${reference} is pinned by digest, and an image placed from an archive cannot keep its ` +
        `digest — a repo digest only exists for an image a registry served.\n` +
        `  Placing it would load an image with no name, and a container declaring that digest ` +
        `would try to reach a registry.\n` +
        `  Place it by tag. What survives being loaded is the image's own ID, which is what a ` +
        `declaration names it by (mesh-host: an image the machine already holds).`,
    );
  }

  const tar = join(tmpdir(), `mesh-lab-image-${process.pid}-${Date.now()}.tar`);

  // Exported from whatever this workstation has. The lab places images; it does not fetch
  // them, so an image nobody pulled here is an error rather than a download.
  const saved = await local("docker", ["save", reference, "-o", tar], 600_000);
  if (!saved.ok) {
    throw new PlacementError(
      machine,
      `cannot export ${reference} from this workstation: ${saved.stderr.trim()}\n` +
        `  The lab places images it already has, and never fetches on a scenario's behalf.\n` +
        `  Pull it here first, then raise.`,
    );
  }

  try {
    await waitForRuntime(instanceName, machine);
    await incus(["file", "push", tar, `${instanceName}/tmp/image.tar`], 900_000);

    // Read back what the runtime says, not that the push returned. A file arriving is not an
    // image present, and `docker load` names what it loaded — so that is what is checked.
    const loaded = await incusOk(
      ["exec", instanceName, "--", "docker", "load", "-i", "/tmp/image.tar"],
      600_000,
    );
    // `Loaded image:` and `Loaded image ID:` are different outcomes and only one is useful.
    // Matching the shorter string accepted both, so an archive that loaded as a dangling
    // image reported success — a check that passes on the wrong thing.
    if (!loaded?.includes("Loaded image:")) {
      throw new PlacementError(
        machine,
        `${reference} was pushed to ${machine} and did not arrive as a usable image.\n` +
          `  The runtime said: ${loaded?.trim() || "nothing"}\n` +
          `  'Loaded image ID:' means it loaded with no name, which nothing can run by name.`,
      );
    }
    log(`  placed ${reference} on ${machine}`);
  } finally {
    await unlink(tar).catch(() => {});
  }
}

/** Run something on THIS workstation. The incus client only speaks to incus. */
function local(
  command: string,
  args: string[],
  timeoutMs: number,
): Promise<{ ok: boolean; stdout: string; stderr: string }> {
  // The third and last place the lab runs an external program (novox/hq 04-ISSUES/024).
  // `docker save` of a large image is the slowest single thing a raise does.
  return around(`${command} ${shorten(args)}`, () => runLocal(command, args, timeoutMs), {
    heartbeatMs: 15_000,
  });
}

function runLocal(
  command: string,
  args: string[],
  timeoutMs: number,
): Promise<{ ok: boolean; stdout: string; stderr: string }> {
  return new Promise((resolve) => {
    const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
    let stdout = "";
    let stderr = "";
    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
    child.stdout.on("data", (d) => (stdout += d));
    child.stderr.on("data", (d) => (stderr += d));
    child.on("error", (err) => {
      clearTimeout(timer);
      resolve({ ok: false, stdout, stderr: err.message });
    });
    child.on("close", (code) => {
      clearTimeout(timer);
      if (code !== 0) {
        log.debug(`  exit ${code}: ${shorten([stderr.trim() || "(nothing on stderr)"], 400)}`);
      }
      resolve({ ok: code === 0, stdout, stderr });
    });
  });
}

/**
 * Wait until the machine's container runtime will answer, and say why if it will not.
 *
 * **A stall must become a failure with a reason.** `docker load` against a daemon that is not
 * running does not fail — the socket exists and is socket-activated, so the client blocks while
 * systemd tries to start a service that may be queued behind something. That turned a
 * misconfigured link into a thirty-five minute silence and then a timeout naming the wrong
 * thing entirely (novox/hq 04-ISSUES/024).
 *
 * The one cause seen in practice is named in the message, because it is not guessable from
 * "docker did not start": these machines have no DHCP by design, so a link that networkd was
 * never told how to configure leaves `network-online.target` unreachable for ever, and Docker
 * is ordered after it.
 */
async function waitForRuntime(instanceName: string, machine: string): Promise<void> {
  const deadline = Date.now() + 120_000;
  while (Date.now() < deadline) {
    if (await succeeds(["exec", instanceName, "--", "docker", "info"], 20_000)) return;
    await new Promise((r) => setTimeout(r, 2_000));
  }

  // What it was waiting for, asked once, so the report names the cause rather than the symptom.
  const jobs = (await incusOk(
    ["exec", instanceName, "--", "systemctl", "list-jobs", "--no-pager"], 20_000,
  )) ?? "";
  const blocked = jobs.includes("network-online.target") || jobs.includes("wait-online");

  throw new PlacementError(
    machine,
    `the container runtime on ${machine} did not answer within 120s, so an image cannot be ` +
      `placed into it.\n` +
      (blocked
        ? `  Docker is queued behind network-online.target, which is waiting for a link that ` +
          `systemd-networkd was never told how to configure. These machines have no DHCP by ` +
          `design, so that wait never ends — give the link a .network unit rather than an ` +
          `address set by hand.\n`
        : "") +
      `  systemd is waiting on:\n${jobs.trim() || "    (it said nothing)"}`,
  );
}

// --- the mesh's own images ----------------------------------------------------------------------

/**
 * Which machine is handed which of the mesh's own images.
 *
 * A machine that says nothing gets the lot, which is right for a one-machine bed. A machine that
 * lists some gets those. This is where a workstation running one small module stops paying for
 * forty runtimes it will never start.
 */
export function planHeldImages(scenario: Scenario): { machine: string; images: string[] }[] {
  const all = scenario.images ?? [];
  if (all.length === 0) return [];
  return Object.entries(scenario.machines)
    .map(([machine, spec]) => ({ machine, images: spec.images ?? all }))
    .filter((plan) => plan.images.length > 0);
}

/**
 * Put the mesh's own images onto the machines that need them, and say what they are now called.
 *
 * **This is what replaced the lab's registry**, and the difference is the whole point. A registry
 * inside the scenario served every image — third-party ones included — from an address that exists
 * in no production mesh, so a bootstrap that could only work against it went green here and would
 * have failed anywhere else. There is no such registry now: third-party images are pulled from the
 * internet over each machine's `egress` uplink, and the mesh's own arrive the way they arrive on an
 * operator's machine — somebody built them and put them there.
 *
 * The reference a declaration then uses is the image's **own ID**, the digest of its configuration.
 * `docker load` preserves it, so the name is identical on the workstation that built the image and
 * on every machine handed a copy — immutable, unforgeable, and requiring nothing to have served it
 * (mesh-host, *an image may be named by the digest of its own configuration*).
 *
 * Read back on both sides. The ID is taken from the workstation and then CONFIRMED on the machine,
 * because a load that lands a different image than the one exported is exactly the silent fault
 * this lab exists to catch — and the reference is what every manifest will be rewritten to.
 */
export async function loadHeldImages(
  scenario: Scenario,
  machineNames: Map<string, string>,
  log: (message: string) => void = () => {},
): Promise<HeldImage[]> {
  const plans = planHeldImages(scenario);
  if (plans.length === 0) return [];

  const held: HeldImage[] = [];

  // Validate everything first, so a scenario naming something it should not is refused before any
  // machine is touched rather than halfway through a transfer.
  for (const requested of scenario.images ?? []) {
    if (!mustBeHandedOver(requested)) {
      throw new Error(
        `images: '${requested}' is not one of the mesh's own images. It is pulled from the ` +
          `internet by the machine that needs it, not loaded from this workstation.`,
      );
    }
    const onThisWorkstation = (await local(
      "docker", ["image", "inspect", "--format", "{{.Id}}", requested], 60_000,
    )).stdout.trim();
    if (!/^sha256:[0-9a-f]{64}$/.test(onThisWorkstation)) {
      throw new Error(
        `${requested} is not on this workstation, so there is nothing to hand the machines.\n` +
          `  It is one of the mesh's own images and exists in no registry — nothing can pull it.\n` +
          `  Build it first (mesh-controller's \`make image …\`, or scripts/build-module-runtime.sh).`,
      );
    }
  }

  // **One archive per machine, not one per image.**
  //
  // These images are almost entirely the same bytes: the base they share is 227 MB and a module's
  // own code is a few. Exported one at a time, that base is written, pushed and loaded once per
  // image — for a bed stocking thirty-odd of them, the same 227 MB crosses thirty-odd times, and
  // measured on this workstation the whole set came to 9.7 GB. Exported together it is written
  // once: two images alone dedupe by 45%, and the saving grows with the count.
  //
  // This is the slowest thing a raise does, and a four-machine bed has been exceeding its own
  // ninety-minute limit while still copying.
  const perMachine = new Map<string, string[]>();
  for (const requested of scenario.images ?? []) {
    for (const plan of plans) {
      if (!plan.images.includes(requested)) continue;
      const list = perMachine.get(plan.machine) ?? [];
      list.push(requested);
      perMachine.set(plan.machine, list);
    }
  }

  // What each machine ended up calling each image, so the disagreement check below still sees
  // every machine's answer for every image.
  const reported = new Map<string, Map<string, string>>();

  for (const [machine, images] of perMachine) {
    const name = machineNames.get(machine);
    if (!name || images.length === 0) continue;

    const tar = join(tmpdir(), `mesh-lab-held-${process.pid}-${Date.now()}.tar`);
    const saved = await local("docker", ["save", ...images, "-o", tar], 1_800_000);
    if (!saved.ok) {
      await unlink(tar).catch(() => {});
      throw new Error(
        `cannot export ${images.length} image(s) for ${machine} from this workstation: ` +
          saved.stderr.trim(),
      );
    }

    try {
      await waitForRuntime(name, machine);
      await incus(["file", "push", tar, `${name}/tmp/held.tar`], 1_800_000);
      const loaded = await incusOk(
        ["exec", name, "--", "docker", "load", "-i", "/tmp/held.tar"], 1_800_000,
      );
      if (!loaded?.includes("Loaded image")) {
        throw new PlacementError(
          machine,
          `${images.length} image(s) were pushed to ${machine} and did not load.\n` +
            `  The runtime said: ${loaded?.trim() || "nothing"}`,
        );
      }

      for (const requested of images) {
        // What this machine calls it, asked of the tag it was just loaded under. An image id is
        // the digest of a configuration the receiving runtime rewrites, so it is read back rather
        // than predicted from here.
        const there = (await incusOk(
          ["exec", name, "--", "docker", "image", "inspect", "--format", "{{.Id}}", requested],
          120_000,
        ))?.trim() ?? "";
        if (!/^sha256:[0-9a-f]{64}$/.test(there)) {
          throw new PlacementError(
            machine,
            `${requested} loaded onto ${machine} and the runtime will not say what it holds.\n` +
              `  It answered '${there || "nothing"}'.\n` +
              `  Nothing serves this image, so a manifest naming it has only what the machine ` +
              `itself reports — and there is nothing to fall back to.`,
          );
        }
        const seen = reported.get(requested) ?? new Map<string, string>();
        seen.set(machine, there);
        reported.set(requested, seen);
      }
      await succeeds(["exec", name, "--", "rm", "-f", "/tmp/held.tar"], 60_000);
    } finally {
      await unlink(tar).catch(() => {});
    }
  }

  // **A manifest carries one reference, so the machines must agree on it.** They are built from one
  // base and load one archive, so they do; if that ever stops being true the image cannot be named
  // from a catalogue at all, and that is worth stopping for rather than rewriting to whichever
  // machine answered last.
  for (const requested of scenario.images ?? []) {
    const seen = reported.get(requested);
    if (!seen || seen.size === 0) continue;
    const entries = [...seen];
    const first = entries[0];
    if (!first) continue;
    const [firstMachine, reference] = first;
    for (const [machine, there] of seen) {
      if (there === reference) continue;
      throw new PlacementError(
        machine,
        `${requested} is ${there} on ${machine} and ${reference} on ${firstMachine}.\n` +
          `  One manifest cannot name both, and this image exists in no registry to be named by ` +
          `instead. The machines' runtimes differ in a way that changes how they store what they ` +
          `are given.`,
      );
    }
    held.push({ requested, repository: repositoryOf(requested), reference });
    log(`  ${requested} → ${reference.slice(0, 19)}… on ${[...seen.keys()].join(", ")}`);
  }

  return held;
}

/**
 * Install the host's systemd packaging, so the installer supervises it as a service rather than a
 * background process — the path a real machine takes, and the only one that survives a reboot. The
 * lab places the binary at HOST_PATH; the shipped launcher runs $MESH_HOST_BIN (default
 * /usr/bin/nox-mesh-host), so a symlink points that at the binary rather than editing the packaged
 * unit. The installer's --host-service then starts AND enables it (novox/hq ADR 0005).
 */
export async function installHostService(
  instanceName: string,
  machine: string,
  packagingDir: string,
  logMessage: (message: string) => void = () => {},
): Promise<void> {
  const launcher = join(packagingDir, "nox-mesh-host-launch");
  const unit = join(packagingDir, "nox-mesh-host.service");
  await incus(["exec", instanceName, "--", "mkdir", "-p", "/usr/lib/nox-mesh-host"], 60_000);
  await incus(["file", "push", launcher, `${instanceName}/usr/lib/nox-mesh-host/launch`, "--mode", "0755"], 120_000);
  await incus(["file", "push", unit, `${instanceName}/etc/systemd/system/nox-mesh-host.service`, "--mode", "0644"], 120_000);
  await incus(["exec", instanceName, "--", "ln", "-sf", HOST_PATH, "/usr/bin/nox-mesh-host"], 60_000);
  await incus(["exec", instanceName, "--", "systemctl", "daemon-reload"], 60_000);
  logMessage(`installed nox-mesh-host.service on ${machine}`);
}