From d637c77f0852a262e8a0c4890b40dda5f398fe29 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 11 Sep 2026 00:02:56 +0200 Subject: [PATCH] An image id belongs to the machine holding it, so ask the machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The id is the digest of an image's configuration, and a runtime rewrites that configuration as it loads: this workstation saves in one format, the machines store it in another, and the same bytes arrive under a different name. Measured, not assumed — b86bb81c here, 2dc21904 there. So it is read back from the machine instead of predicted from here. Predicting it failed at the only moment it mattered: every manifest would have been rewritten to a reference no machine holds, and these images exist in no registry, so each apply would have stopped at a pull that cannot succeed. A manifest carries one reference, so machines that disagree stop the raise rather than having one of them silently win. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- src/lifecycle/place.ts | 57 +++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/lifecycle/place.ts b/src/lifecycle/place.ts index 4414d12..32b4640 100644 --- a/src/lifecycle/place.ts +++ b/src/lifecycle/place.ts @@ -458,17 +458,28 @@ export async function loadHeldImages( const wanted = plans.filter((plan) => plan.images.includes(requested)).map((p) => p.machine); if (wanted.length === 0) continue; - const id = (await local( + const onThisWorkstation = (await local( "docker", ["image", "inspect", "--format", "{{.Id}}", requested], 60_000, )).stdout.trim(); - if (!/^sha256:[0-9a-f]{64}$/.test(id)) { + if (!/^sha256:[0-9a-f]{64}$/.test(onThisWorkstation)) { throw new Error( `${requested} is not on this workstation, so there is nothing to hand the machines.\n` + ` It is one of the mesh's own images and exists in no registry — nothing can pull it.\n` + ` Build it first (mesh-control's \`make image …\`, or scripts/build-module-runtime.sh).`, ); } - const image: HeldImage = { requested, repository: repositoryOf(requested), reference: id }; + + // **An image id belongs to the runtime holding it, and does not survive the journey.** It is + // the digest of the image's *configuration*, and a runtime rewrites that configuration as it + // loads: this workstation saves in one format and the machine's older runtime stores it in + // another, so the same bytes arrive under a different name. Measured, not assumed — the same + // image was b86bb81c… here and 2dc21904… on the machine. + // + // So the id is READ BACK from the machine rather than predicted from here. Predicting it is + // what the earlier version did, and it failed at the only useful moment: the manifests would + // have been rewritten to a reference no machine holds, and nothing serves these images, so + // every apply would have stopped at the container with a pull that cannot succeed. + let reference = ""; // Exported once, handed to each machine that asked for it. The archive is the expensive part // and it does not depend on the destination. @@ -495,19 +506,35 @@ export async function loadHeldImages( ` The runtime said: ${loaded?.trim() || "nothing"}`, ); } - // The reference every manifest is about to be rewritten to, confirmed present under - // exactly that name. Asking for the tag would prove the load happened; asking for the ID - // proves the thing a declaration will name is the thing that is there. + // What this machine calls it, asked of the tag it was just loaded under. This is the + // reference every manifest naming this image will be rewritten to. const there = (await incusOk( - ["exec", name, "--", "docker", "image", "inspect", "--format", "{{.Id}}", id], 120_000, - ))?.trim(); - if (there !== id) { + ["exec", name, "--", "docker", "image", "inspect", "--format", "{{.Id}}", requested], + 120_000, + ))?.trim() ?? ""; + if (!/^sha256:[0-9a-f]{64}$/.test(there)) { throw new PlacementError( machine, - `${requested} loaded onto ${machine} and is not there as ${id}.\n` + - ` The runtime answered '${there || "nothing"}'.\n` + - ` Every manifest naming this image would be rewritten to a reference the machine ` + - `does not hold, and nothing serves it — so the apply would stop at the container.`, + `${requested} loaded onto ${machine} and the runtime will not say what it holds.\n` + + ` It answered '${there || "nothing"}'.\n` + + ` Nothing serves this image, so a manifest naming it has only what the machine ` + + `itself reports — and there is nothing to fall back to.`, + ); + } + // **A manifest carries one reference, so the machines must agree on it.** They are built + // from one base image and load one archive, so they do; if that ever stops being true the + // image cannot be named at all from a catalogue, and that is worth stopping for rather + // than rewriting to whichever machine answered last. + if (!reference) { + reference = there; + } else if (reference !== there) { + throw new PlacementError( + machine, + `${requested} is ${there} on ${machine} and ${reference} on a machine already ` + + `loaded.\n` + + ` One manifest cannot name both, and this image exists in no registry to be ` + + `named by instead. The machines' runtimes differ in a way that changes how they ` + + `store what they are given.`, ); } await succeeds(["exec", name, "--", "rm", "-f", "/tmp/held.tar"], 60_000); @@ -516,8 +543,8 @@ export async function loadHeldImages( await unlink(tar).catch(() => {}); } - held.push(image); - log(` ${requested} → ${id.slice(0, 19)}… on ${wanted.join(", ")}`); + held.push({ requested, repository: repositoryOf(requested), reference }); + log(` ${requested} → ${reference.slice(0, 19)}… on ${wanted.join(", ")}`); } return held; }