Merge pull request 'Five postgres consumers say which provision their check exercises (hq issue 145, to-be 48 Phase E)' (#289) from fix/145-a-consumer-is-held-under-its-provider into main

Any route umami does not serve answers 404 too, as an HTML page. The
share route answers JSON with code not-found only after the database
was asked; with the database down it answers 500.

RELEASE-57.md

@@ -1,6 +0,0 @@
-# Release 57: config and provisioning
-
-This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
-merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
-delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
-and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.

checks/predefined/go.mod

@@ -1,3 +0,0 @@
-module git.novox.be/novox/mesh-catalog/checks/predefined
-
-go 1.22

checks/predefined/main.go

@@ -1,153 +0,0 @@
-// Command predefined refuses a change to predefined.json that removes a key a module still names (novox/hq ADR
-// 0348 clause 7, design 57 package K2, issue 661).
-//
-// Mesh config and node config hold values only for the keys on the catalogue's list of predefined keys, at the
-// root of the repository; a module's config value may fall back to one, `"predefined": "<key>"`. A key removed
-// while a manifest still names it would leave that module falling back to nothing: the controller refuses its
-// build then, at the next merge that touches it. So the removal is refused here, in the pull request that makes
-// it, naming each module that names the key.
-//
-// **What it cannot know.** Whether a key still holds a value at the mesh or a node is mesh-config's to say (design
-// 57 package K3): it holds the values, and the catalogue's check reads no store. Until mesh-config refuses that
-// itself, removing a key nobody names is let through here, and said.
-//
-// The list it compares with is the trunk's at the merge base, given with -base; without one, nothing was removed.
-// The controller the mesh runs reads the list itself (type, validation, every manifest against it) in the
-// module check; this check reads only its keys.
-//
-//	go run . [-base <the trunk's predefined.json>] <catalogue root>
-//
-// Exits 1 on a finding.
-package main
-
-import (
-	"encoding/json"
-	"errors"
-	"flag"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-)
-
-// File is the list's file at the root of the catalogue.
-const File = "predefined.json"
-
-// Keys is the keys of a list, read as a JSON object; an absent file has none.
-func Keys(path string) (map[string]bool, error) {
-	raw, err := os.ReadFile(path)
-	if errors.Is(err, fs.ErrNotExist) {
-		return map[string]bool{}, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	var keys map[string]json.RawMessage
-	if err := json.Unmarshal(raw, &keys); err != nil {
-		return nil, fmt.Errorf("%s is not a JSON object of keys: %w", path, err)
-	}
-	out := map[string]bool{}
-	for k := range keys {
-		out[k] = true
-	}
-	return out, nil
-}
-
-// Named is, per predefined key, the modules whose manifests name it: under `config`, or its alias `settings`.
-func Named(root string) (map[string][]string, error) {
-	manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
-	if err != nil {
-		return nil, err
-	}
-	sort.Strings(manifests)
-	named := map[string][]string{}
-	for _, path := range manifests {
-		raw, err := os.ReadFile(path)
-		if err != nil {
-			return nil, err
-		}
-		var m struct {
-			Module   string                     `json:"module"`
-			Config   map[string]json.RawMessage `json:"config"`
-			Settings map[string]json.RawMessage `json:"settings"`
-		}
-		if err := json.Unmarshal(raw, &m); err != nil {
-			return nil, fmt.Errorf("%s: %w", path, err)
-		}
-		for _, declared := range []map[string]json.RawMessage{m.Config, m.Settings} {
-			for _, d := range declared {
-				var value struct {
-					Predefined string `json:"predefined"`
-				}
-				if json.Unmarshal(d, &value) == nil && value.Predefined != "" {
-					named[value.Predefined] = append(named[value.Predefined], m.Module)
-				}
-			}
-		}
-	}
-	return named, nil
-}
-
-// Removed is each key the base list holds and the tree's does not, with the modules still naming it; a removed
-// key nobody names is answered with none.
-func Removed(base, tree map[string]bool, named map[string][]string) map[string][]string {
-	out := map[string][]string{}
-	for k := range base {
-		if !tree[k] {
-			out[k] = named[k]
-		}
-	}
-	return out
-}
-
-func main() {
-	basePath := flag.String("base", "", "the trunk's predefined.json at the merge base")
-	flag.Parse()
-	if flag.NArg() != 1 {
-		fmt.Fprintln(os.Stderr, "usage: predefined [-base <the trunk's predefined.json>] <catalogue root>")
-		os.Exit(2)
-	}
-	root := flag.Arg(0)
-	tree, err := Keys(filepath.Join(root, File))
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	if *basePath == "" {
-		fmt.Printf("%s: %d key(s); no trunk list given, so none was removed\n", File, len(tree))
-		return
-	}
-	base, err := Keys(*basePath)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	named, err := Named(root)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	removed := Removed(base, tree, named)
-	keys := make([]string, 0, len(removed))
-	for k := range removed {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	failed := false
-	for _, k := range keys {
-		if modules := removed[k]; len(modules) > 0 {
-			failed = true
-			fmt.Printf("%s: the key %q is removed and still named by %s: move each off it first (novox/hq ADR 0348 "+
-				"clause 7)\n", File, k, strings.Join(modules, ", "))
-		} else {
-			fmt.Printf("%s: the key %q is removed and no module names it; whether the mesh or a node still holds a "+
-				"value for it is mesh-config's to refuse, not this check's\n", File, k)
-		}
-	}
-	if failed {
-		os.Exit(1)
-	}
-	fmt.Printf("%s: %d key(s), none removed that a module names\n", File, len(tree))
-}

checks/predefined/main_test.go

@@ -1,100 +0,0 @@
-package main
-
-import (
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"testing"
-)
-
-// A catalogue whose dunst falls back to font-size, and the trunk's list holding font-size and
-// agent-managed-settings.
-func aCatalogue(t *testing.T, list string) (root, base string) {
-	t.Helper()
-	root = t.TempDir()
-	write := func(path, text string) {
-		t.Helper()
-		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.WriteFile(path, []byte(text), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-	write(filepath.Join(root, "modules", "dunst", "module.json"), `{"module": "dunst", "config": {"font-size":
-	  {"type": "number", "description": "points", "default": 10, "predefined": "font-size"}}}`)
-	write(filepath.Join(root, "modules", "claude-code", "module.json"), `{"module": "claude-code", "settings": {
-	  "managed_settings": {"kind": "preference", "default": "{}", "why": "none"}}}`)
-	write(filepath.Join(root, "modules", "plain", "module.json"), `{"module": "plain"}`)
-	if list != "" {
-		write(filepath.Join(root, File), list)
-	}
-	base = filepath.Join(t.TempDir(), "base.json")
-	write(base, `{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}}`)
-	return root, base
-}
-
-func run(t *testing.T, args ...string) (string, error) {
-	t.Helper()
-	out, err := exec.Command("go", append([]string{"run", "."}, args...)...).CombinedOutput()
-	return string(out), err
-}
-
-func TestARemovalOfAKeyAModuleStillNamesIsRefused(t *testing.T) {
-	// The planted removal: font-size goes while dunst still falls back to it.
-	root, base := aCatalogue(t, `{"agent-managed-settings": {"type": "object"}}`)
-	out, err := run(t, "-base", base, root)
-	if err == nil || !strings.Contains(out, `the key "font-size" is removed and still named by dunst`) {
-		t.Fatalf("a removal of a named key passed:\n%s", out)
-	}
-}
-
-func TestARemovalNobodyNamesPassesAndSaysWhatItCannotKnow(t *testing.T) {
-	root, base := aCatalogue(t, `{"font-size": {"type": "number"}}`)
-	out, err := run(t, "-base", base, root)
-	if err != nil || !strings.Contains(out, `"agent-managed-settings" is removed and no module names it`) ||
-		!strings.Contains(out, "mesh-config's to refuse") {
-		t.Fatalf("a removal nobody names: %v\n%s", err, out)
-	}
-}
-
-func TestAListUnchangedOrGrownPasses(t *testing.T) {
-	for _, list := range []string{
-		`{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}}`,
-		`{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}, "accent": {"type": "string"}}`,
-	} {
-		root, base := aCatalogue(t, list)
-		if out, err := run(t, "-base", base, root); err != nil || !strings.Contains(out, "none removed") {
-			t.Fatalf("%s: %v\n%s", list, err, out)
-		}
-	}
-	// No trunk list given: nothing was removed.
-	root, _ := aCatalogue(t, `{}`)
-	if out, err := run(t, root); err != nil || !strings.Contains(out, "none was removed") {
-		t.Fatalf("without a base: %v\n%s", err, out)
-	}
-}
-
-func TestTheFileRemovedWholeIsARemovalOfEveryKey(t *testing.T) {
-	root, base := aCatalogue(t, "")
-	out, err := run(t, "-base", base, root)
-	if err == nil || !strings.Contains(out, `"font-size" is removed and still named by dunst`) {
-		t.Fatalf("the list removed whole passed:\n%s", out)
-	}
-}
-
-func TestNamedReadsBothForms(t *testing.T) {
-	root, _ := aCatalogue(t, "")
-	if err := os.WriteFile(filepath.Join(root, "modules", "plain", "module.json"), []byte(`{"module": "plain",
-	  "settings": {"size": {"predefined": "font-size"}}}`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	named, err := Named(root)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := strings.Join(named["font-size"], ","); got != "dunst,plain" {
-		t.Fatalf("font-size named by %s", got)
-	}
-}

health-undeclared

@@ -1,4 +1,4 @@
-66
+62
 The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
 rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
 one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running

merge-check.sh

@@ -33,10 +33,6 @@
 #   6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
 #      replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
 #      the nested ones the controller cannot protect alone yet, and only shrinks.
-#   7. the list of predefined keys (novox/hq ADR 0348 clause 7): a change to predefined.json removes no key a
-#      module still names, judged against the trunk's copy at the merge base. Whether a removed key still holds a
-#      value at the mesh or a node is mesh-config's to refuse; this check reads no store. The list itself and
-#      every manifest against it are judged by the controller's module check in step 1.
 set -eu
 
 (cd checks/words && go test -count=1 . && go run . ../..)
@@ -55,18 +51,6 @@ else
 	echo "NOT COMPARED WITH THE TRUNK: no merge base with $secrets_ref; checks/secrets/known is held to the list it was first written with"
 fi
 (cd checks/secrets && go test -count=1 . && if [ -n "$secrets_base" ]; then go run . -base "$secrets_base" ../..; else go run . ../..; fi)
-# The list of predefined keys (novox/hq ADR 0348 clause 7): no key removed that a module still names. The trunk's
-# list at the merge base is what it is judged against; a trunk without the file had none.
-predefined_base=""
-if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then
-	predefined_base=$(mktemp)
-	if ! git show "$merge_base:predefined.json" >"$predefined_base" 2>/dev/null; then
-		echo '{}' >"$predefined_base"
-	fi
-else
-	echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; no removal from predefined.json can be seen"
-fi
-(cd checks/predefined && go test -count=1 . && if [ -n "$predefined_base" ]; then go run . -base "$predefined_base" ../..; else go run . ../..; fi)
 # The lists that narrow the paths check may not grow unseen: the base branch's copies are what they are
 # judged against (MESH_CHECK_BASE, main when unset), read from the clone the check runs in.
 base_lists=""

modules/gitea/module.json

@@ -149,6 +149,13 @@
     {
       "id": "server",
       "type": "container",
+      "health": {
+        "kind": "http",
+        "endpoint": "web",
+        "path": "/api/healthz",
+        "status": 200,
+        "needs": "postgres-database"
+      },
       "name": "gitea",
       "image": "gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef",
       "env": {

modules/influxdb/module.json

@@ -26,24 +26,12 @@
       "why": "queries, writes and the web UI, over http; consumers granted influxdb-api sign in with the mesh's credential, and a name is a route grant"
     }
   ],
-  "settings": {
-    "org": {
-      "kind": "preference",
-      "default": "mesh",
-      "why": "the organisation influxdb-api names to its consumers; mesh is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11)"
-    },
-    "bucket": {
-      "kind": "preference",
-      "default": "default",
-      "why": "the bucket influxdb-api names to its consumers; default is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11)"
-    }
-  },
   "serves": {
     "influxdb-api": {
       "scheme": "http",
       "port": 8086,
-      "org": "${setting:org}",
-      "bucket": "${setting:bucket}"
+      "org": "mesh",
+      "bucket": "default"
     }
   },
   "receives": {
@@ -136,13 +124,7 @@
       "path": "${dir:mesh-state}/config.json",
       "mode": "0600",
       "content": "{}\n",
-      "merge": "json",
-      "takes-settings": [
-        "bucket",
-        "org",
-        "url"
-      ],
-      "trusted": true
+      "merge": "json"
     }
   ],
   "requires": [

modules/lab/test/base.test.ts

@@ -0,0 +1,198 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import {
+  BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath,
+  describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase,
+  type BaseImage,
+} from "../tools/runs.ts";
+
+// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a
+// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for
+// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it.
+
+const now = new Date("2026-10-14T01:00:00Z");
+const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([
+  { aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" },
+  { aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" },
+]);
+
+test("the base image's age is read from when this machine made it", () => {
+  const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
+  assert.equal(b.exists, true);
+  assert.equal(b.built, "2026-08-31T09:00:00.000Z");
+  assert.equal(b.ageDays, 43);
+  assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/);
+});
+
+test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => {
+  const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
+  const reason = runRefusal(stale);
+  assert.ok(reason, "a 43-day-old base image was not refused");
+  assert.match(reason, /older than the lab's bound of 14 days/);
+  assert.match(reason, /lab_base \{"rebuild": true\}/);
+  assert.equal(staleBase(stale), reason);
+});
+
+test("an image within the bound runs", () => {
+  const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now);
+  assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS);
+  assert.equal(runRefusal(fresh), undefined);
+});
+
+test("a missing image, or one whose age cannot be read, is said and not refused", () => {
+  const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now);
+  assert.equal(none.exists, false);
+  assert.equal(runRefusal(none), undefined);
+  assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/);
+  const unread = baseImageFrom(undefined, now);
+  assert.equal(runRefusal(unread), undefined);
+  assert.match(describeBase(unread), /its age is not known/);
+});
+
+test("no run starts while the base image is rebuilt", () => {
+  const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now);
+  const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" });
+  assert.match(reason ?? "", /is being rebuilt \(base-x/);
+});
+
+test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => {
+  const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials");
+  assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
+  assert.match(s, /src\/cli.ts base build/);
+  assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone");
+});
+
+test("a rebuild is known by its status, and is not a run", () => {
+  const work = mkdtempSync(join(tmpdir(), "lab-base-"));
+  try {
+    mkdirSync(join(work, "base-1"));
+    writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" }));
+    mkdirSync(join(work, "base-2"));
+    // A build whose process is gone while it says building was lost, and holds nothing up.
+    writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 }));
+    assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]);
+    assert.equal(buildingBase(work), undefined);
+    writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid }));
+    assert.equal(buildingBase(work)?.id, "base-2");
+    assert.deepEqual(listRuns(work), []);
+    assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8"));
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which
+// waits, and each refused while the other was going; asked at once, both passed their checks before either
+// wrote its status. The registry is asked first now, and the check and the start are one step.
+function aLab(): string {
+  return mkdtempSync(join(tmpdir(), "lab-launch-"));
+}
+const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 };
+const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20));
+const launch = { registry: slowRegistry, base: () => fresh };
+function beginRun(work: string, id: string) {
+  return () => {
+    mkdirSync(join(work, id));
+    const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid };
+    writeFileSync(join(work, id, "status.json"), JSON.stringify(s));
+    return s;
+  };
+}
+function beginBuild(work: string, id: string) {
+  return () => {
+    mkdirSync(join(work, id));
+    const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid };
+    writeFileSync(join(work, id, "status.json"), JSON.stringify(b));
+    return b;
+  };
+}
+
+test("a run and a rebuild asked at once: exactly one starts", async () => {
+  for (const runFirst of [true, false]) {
+    const work = aLab();
+    try {
+      // Both asked before either answers, in one order and then the other.
+      const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) };
+      const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()])
+        : (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited<ReturnType<typeof ask.run>>, Awaited<ReturnType<typeof ask.build>>];
+      assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`);
+      assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1);
+    } finally {
+      rmSync(work, { recursive: true, force: true });
+    }
+  }
+});
+
+test("two runs, or two rebuilds, asked at once: one of each starts", async () => {
+  const work = aLab();
+  try {
+    const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]);
+    assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]);
+    const other = aLab();
+    const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]);
+    assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]);
+    rmSync(other, { recursive: true, force: true });
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a run is refused on a stale base, and starts nothing", async () => {
+  const work = aLab();
+  try {
+    const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 };
+    const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a"));
+    assert.equal(r.started, false);
+    assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/);
+    assert.deepEqual(listRuns(work), []);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => {
+  const work = aLab();
+  try {
+    mkdirSync(join(work, "base-x"));
+    writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" }));
+    mkdirSync(join(work, "run-x"));
+    writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} }));
+    assert.equal(listBaseBuilds(work)[0]?.state, "lost");
+    assert.equal(buildingBase(work), undefined);
+    assert.equal(readStatus(work, "run-x")?.state, "lost");
+    assert.equal(running(work), undefined);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => {
+  const work = aLab();
+  try {
+    for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x");
+    const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort();
+    assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort());
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => {
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/);
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/);
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/);
+  assert.equal(baseProblem(fresh), undefined);
+});
+
+// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read).
+test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => {
+  const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8");
+  assert.match(index, /await launchRun\(work,/);
+  assert.match(index, /await launchRebuild\(work,/);
+  assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/);
+  assert.match(index, /if \(args\.rebuild !== true\) return reading;/);
+});

modules/lab/test/runs.test.ts

@@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
-import { installPlan, installStep, NPM_REPOSITORIES, REPOSITORIES, script } from "../tools/runs.ts";
+import { installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts";
 
 // Defends novox/hq issue 624: lab_run ran `npm ci` at the root of every checkout it installs, and
 // mesh-tools keeps its package in node-tools/ with no lock file at its root, so every run failed at its
@@ -381,3 +381,108 @@ test("a forge setting that is not http(s) trusts no origin of its own", async ()
     packument("ftp://novox.internal/api/packages/novox/npm/x.tgz")).get, root);
   assert.match(viaRoot.problem ?? "", /neither/);
 });
+
+// Defends novox/hq issue 693: a run whose only test skipped as NOT REACHED read "0 passed, 0 failed", exit 0,
+// and lab_status said passed. The log's tail, as run-2026-10-14T00-21-44-014Z on shanks wrote it.
+const oldLog = [
+  "STOPPED at step 16 of 24, base:          \t\t\t\tMake sure you have all the referenced content and try again.",
+  "\ufe63 genesis adopts its bus as the nats module in place: one bus, its streams kept, the node connected (588152.016874ms) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:",
+  "         \t\t\t\tMake sure you have all the referenced content and try again.",
+  "NotFound: content digest sha256:e2de96513ba9: not found",
+  "ℹ tests 1", "ℹ suites 0", "ℹ pass 0", "ℹ fail 0", "ℹ cancelled 0", "ℹ skipped 1", "ℹ todo 0",
+  "ℹ duration_ms 590254.163993",
+  "",
+  "recorded: test/integration/genesis-keeps-its-bus.test.ts — 0 passed, 0 failed, against mesh-lab 7ed7da1",
+].join("\n");
+
+// The same run, as the lab's suite says it once it names its skips (mesh-lab, issue 693).
+const newLog = [
+  "ℹ tests 1", "ℹ pass 0", "ℹ fail 0", "ℹ skipped 1",
+  "",
+  "recorded: test/integration/genesis-keeps-its-bus.test.ts — 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1",
+  "NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED",
+  "skipped: genesis adopts its bus — one bus (test/integration/genesis-keeps-its-bus.test.ts) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest",
+].join("\n");
+
+function anEndedRun(log: string, state: "passed" | "failed", exit: number): { work: string; id: string } {
+  const work = mkdtempSync(join(tmpdir(), "lab-runs-"));
+  const id = "run-2026-10-14T00-21-44-014Z";
+  mkdirSync(join(work, id));
+  writeFileSync(join(work, id, "run.log"), log);
+  writeFileSync(join(work, id, "status.json"), JSON.stringify({
+    id, state, exit, started: "2026-10-14T00:21:44.014Z", ended: "2026-10-14T00:33:11.179Z",
+    tests: ["test/integration/genesis-keeps-its-bus.test.ts"], refs: {},
+  }));
+  return { work, id };
+}
+
+test("a run whose only test skipped as NOT REACHED, from a lab of before, reads as failed with the reason", () => {
+  const { work, id } = anEndedRun(oldLog, "passed", 0);
+  const s = readStatus(work, id);
+  assert.equal(s?.state, "failed", JSON.stringify(s));
+  assert.equal(s?.exit, 0);
+  assert.equal(s?.outcome?.passed, 0);
+  assert.equal(s?.outcome?.skipped, 1);
+  assert.equal(s?.outcome?.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+  assert.match(s?.outcome?.skips[0]?.reason ?? "", /^ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:$/);
+  // Kept in the run's record, not only answered.
+  const kept = JSON.parse(readFileSync(join(work, id, "status.json"), "utf8"));
+  assert.equal(kept.state, "failed");
+  assert.equal(kept.outcome.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+  rmSync(work, { recursive: true, force: true });
+});
+
+test("the lab's own NOT PASSED and skipped lines are read, a test name's punctuation and all", () => {
+  const { work, id } = anEndedRun(newLog, "failed", 1);
+  const s = readStatus(work, id);
+  assert.equal(s?.state, "failed");
+  assert.equal(s?.outcome?.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+  assert.deepEqual(s?.outcome?.skips, [{
+    file: "test/integration/genesis-keeps-its-bus.test.ts",
+    test: "genesis adopts its bus — one bus",
+    reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest",
+  }]);
+  rmSync(work, { recursive: true, force: true });
+});
+
+test("a run that passed tests and skipped one for a declared reason stays passed, its skip listed", () => {
+  const log = [
+    "✔ a module lands (12ms)",
+    "﹣ a builder builds (0.1ms) # set MESH_LAB_BUILDER to a built mesh-builder",
+    "ℹ tests 2", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 1",
+  ].join("\n");
+  const { work, id } = anEndedRun(log, "passed", 0);
+  const s = readStatus(work, id);
+  assert.equal(s?.state, "passed");
+  assert.equal(s?.outcome?.notPassed, undefined);
+  assert.equal(s?.outcome?.skips[0]?.reason, "set MESH_LAB_BUILDER to a built mesh-builder");
+  rmSync(work, { recursive: true, force: true });
+});
+
+test("a run that ran no test at all is not passed", () => {
+  assert.equal(outcomeOf("ℹ tests 0\nℹ pass 0\nℹ fail 0\nℹ skipped 0\n").notPassed, "no test passed: none ran");
+});
+
+// Review 254 on issue 693: a test's own output, printed before the suite reports, is not the suite's verdict.
+test("a line a test printed that looks like the suite's is not read as the suite's", () => {
+  const log = [
+    "NOT PASSED: a test printed this",
+    "skipped: a forged skip (test/integration/x.test.ts) # FORGED NOT REACHED",
+    "✔ a module lands (12ms)",
+    "ℹ tests 1", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 0",
+    "",
+    "recorded: test/integration/x.test.ts — 1 passed, 0 failed, 0 skipped, against mesh-lab 7ed7da1",
+  ].join("\n");
+  const o = outcomeOf(log);
+  assert.equal(o.notPassed, undefined, JSON.stringify(o));
+  assert.deepEqual(o.skips, []);
+});
+
+test("not reached is matched in any case", () => {
+  const log = [
+    "✔ a module lands (12ms)",
+    "﹣ the broker step (0.1ms) # the broker step was not reached",
+    "ℹ tests 2", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 1",
+  ].join("\n");
+  assert.equal(outcomeOf(log).notPassed, "1 test(s) skipped as NOT REACHED: their subject was not reached");
+});

modules/lab/tools/index.ts

@@ -9,7 +9,8 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
   checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
-  stop, tail, writeCredential,
+  stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun,
+  launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem,
 } from "./runs.js";
 
 export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
@@ -59,12 +60,18 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
         // repository it clones answers on the forge, and every one it installs has a lock file where the run
         // installs from.
-        // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use
-        // by a running run is not.
-        const busy = running(work);
-        const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined);
-        const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : "";
-        const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...leftovers, ...problems].filter(Boolean).join("\n");
+        // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in
+        // use by a running run or rebuild is not.
+        const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id);
+        const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id)));
+        const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : "";
+        // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot
+        // be read is a problem here, though lab_run refuses only the first.
+        const base = readBaseImage();
+        const unfit = baseProblem(base);
+        if (unfit) problems.push(unfit);
+        const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems]
+          .filter(Boolean).join("\n");
         return { ok: check.status === 0 && problems.length === 0, output };
       },
     },
@@ -96,11 +103,43 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         }
         const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r));
         if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
-        const busy = running(work);
-        if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy };
-        const registry = await registryOf(forge);
-        return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry),
-          registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) };
+        // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698,
+        // review 256): the registry asked first, then the check and the start with nothing awaited between.
+        const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() },
+          (registry) => start(work, forge, tests, refs, credentialOf(forge), registry));
+        if (!launched.started) return launched;
+        return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry,
+          ...(launched.problem ? { problem: launched.problem } : {}) };
+      },
+    },
+    {
+      name: "lab_base",
+      description:
+        "The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " +
+        `age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` +
+        "Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " +
+        "main, in the background, and answers at once; ask again for the build's state and the image's new age.",
+      input: {
+        rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" },
+      },
+      run: async (args) => {
+        const before = readBaseImage();
+        const builds = listBaseBuilds(work).slice(0, 3);
+        const newest = builds[0];
+        const reading = {
+          base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS,
+          ...(baseProblem(before) ? { problem: baseProblem(before) } : {}),
+          builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}),
+        };
+        if (args.rebuild !== true) return reading;
+        const forge = forgeOf();
+        if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
+        // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with
+        // nothing awaited between (review 256).
+        const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before },
+          (registry) => startBaseBuild(work, forge, credentialOf(forge), registry));
+        if (launched.started === false) return { ...reading, started: false, reason: launched.reason };
+        return { ...reading, started: true, build: launched.job };
       },
     },
     {

modules/lab/tools/runs.ts

@@ -6,7 +6,7 @@
 // and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
 // for it, and says it was lost rather than pretending it is still going.
 
-import { spawn } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
@@ -223,16 +223,18 @@ trap ${shellQuote(`${rm}; exit 129`)} HUP`;
 }
 
 /**
- * cleanCredentials removes every run's credential file left in the work directory but the one named — what
- * a crash or a reboot left behind, which no trap could remove. Answers what it removed.
+ * cleanCredentials removes every run's and every base rebuild's credential file left in the work directory
+ * but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it
+ * removed.
  */
-export function cleanCredentials(work: string, keep?: string): string[] {
+export function cleanCredentials(work: string, keep?: string | string[]): string[] {
   if (!existsSync(work)) return [];
+  const kept = new Set([keep ?? []].flat());
   const removed: string[] = [];
   for (const name of readdirSync(work)) {
-    if (!/^\.run-.*\.git-credentials$/.test(name)) continue;
+    if (!/^\.(run|base)-.*\.git-credentials$/.test(name)) continue;
     const path = join(work, name);
-    if (path === keep) continue;
+    if (kept.has(path)) continue;
     rmSync(path, { force: true });
     removed.push(path);
   }
@@ -286,6 +288,117 @@ export interface RunStatus {
   commits?: Record<string, string>;
   exit?: number;
   pid?: number;
+  /** What the suite said of its tests, read from the run's log once it ended (novox/hq issue 693). */
+  outcome?: Outcome;
+}
+
+/** One skipped test, its file and the reason it gave. */
+export interface Skip {
+  file: string;
+  test: string;
+  reason: string;
+}
+
+/**
+ * What a run's suite said of its tests: its totals, each skip with its reason, and — when the run did not
+ * pass although its exit said 0 — why (`notPassed`). Totals the runner did not print are absent.
+ */
+export interface Outcome {
+  passed?: number;
+  failed?: number;
+  skipped?: number;
+  skips: Skip[];
+  notPassed?: string;
+}
+
+/**
+ * outcomeOf reads what a run's log says of its tests (novox/hq issue 693).
+ *
+ * **A run in which no test passed has proven nothing**, and the runner exits 0 when every test skipped:
+ * the genesis bed skipped its only test as "NOT REACHED", the suite printed "0 passed, 0 failed", and the
+ * lab recorded the run as passed. The lab's suite now says it itself — a "NOT PASSED: <why>" line and a
+ * "skipped: <test> (<file>) # <reason>" line per skip after its "recorded:" line, and an exit that is not
+ * 0 — and this reads those lines. For a run of a lab from before (a branch, or an old log), the same rule is
+ * read from the runner's own output: its summary totals, and its skip lines' first line of reason.
+ *
+ * The rule, the lab suite's (mesh-lab src/outcomes.ts): **not passed when no test passed**, or when a test
+ * skipped as NOT REACHED (its subject was not reached); a skip for another reason beside tests that passed
+ * leaves the run passed, its reason listed.
+ */
+export function outcomeOf(log: string): Outcome {
+  const plain = log.replace(/\u001b\[[0-9;]*m/g, "");
+  const total = (word: string) => {
+    // The last summary in the log: the runner's own, once per run.
+    const all = [...plain.matchAll(new RegExp(`^\\s*(?:\u2139|#)\\s*${word}\\s+(\\d+)\\s*$`, "gm"))];
+    const last = all[all.length - 1];
+    return last ? Number(last[1]) : undefined;
+  };
+  const outcome: Outcome = { skips: [] };
+  const passed = total("pass");
+  const failed = total("fail");
+  const skipped = total("skipped");
+  if (passed !== undefined) outcome.passed = passed;
+  if (failed !== undefined) outcome.failed = failed;
+  if (skipped !== undefined) outcome.skipped = skipped;
+
+  // **The suite's own lines are read only after its last "recorded:" line** (novox/hq issue 693, review
+  // 254): a test prints what it likes before the suite reports, and a line of its that looked like the
+  // suite's — "NOT PASSED: …", "skipped: … # NOT REACHED" — would otherwise stand for the suite's verdict.
+  const recorded = [...plain.matchAll(/^recorded: .*$/gm)];
+  const lastRecorded = recorded[recorded.length - 1];
+  const suiteSaid = lastRecorded ? plain.slice((lastRecorded.index ?? 0) + lastRecorded[0].length) : "";
+
+  // The suite's own line, "skipped: <test> (<file>) # <reason>": the file in brackets, a .ts, parts it.
+  for (const m of suiteSaid.matchAll(/^skipped: (.*?) \(([^()]+\.ts)\) # (.*)$/gm)) {
+    outcome.skips.push({ file: m[2] ?? "", test: m[1] ?? "", reason: m[3] === "no reason given" ? "" : m[3] ?? "" });
+  }
+  if (outcome.skips.length === 0) {
+    // A lab from before names no skip itself; its runner's spec line does: "﹣ <test> (<ms>ms) # <reason>".
+    for (const m of plain.matchAll(/^\s*\ufe63 (.*?) \([\d.]+m?s\) # (.*)$/gm)) {
+      const reason = (m[2] ?? "").trim();
+      outcome.skips.push({ file: "", test: m[1] ?? "", reason: reason === "SKIP" ? "" : reason });
+    }
+  }
+
+  const said = suiteSaid.match(/^NOT PASSED: (.*)$/m);
+  if (said) {
+    outcome.notPassed = (said[1] ?? "").trim();
+  } else if ((failed ?? 0) === 0) {
+    // In any case: "not reached" says the same as "NOT REACHED".
+    const unreached = outcome.skips.filter((s) => s.reason.toUpperCase().includes("NOT REACHED")).length;
+    if (passed === 0) {
+      outcome.notPassed = outcome.skips.length > 0 || (skipped ?? 0) > 0
+        ? `no test passed: ${skipped ?? outcome.skips.length} skipped${unreached > 0 ? `, ${unreached} of them as NOT REACHED` : ""}`
+        : "no test passed: none ran";
+    } else if (unreached > 0) {
+      outcome.notPassed = `${unreached} test(s) skipped as NOT REACHED: their subject was not reached`;
+    }
+  }
+  return outcome;
+}
+
+/**
+ * settled is a run that ended, with its outcome read from its log: a run whose exit said 0 and whose suite
+ * proved nothing reads as failed, with why (novox/hq issue 693). Read once and kept in its status, so the
+ * record says it too; a run still going, stopped or lost is left as it is.
+ */
+export function settled(work: string, s: RunStatus): RunStatus {
+  if (s.outcome || !s.ended || (s.state !== "passed" && s.state !== "failed")) return s;
+  let log: string;
+  try {
+    log = readFileSync(join(runDir(work, s.id), "run.log"), "utf8");
+  } catch {
+    return s;
+  }
+  const outcome = outcomeOf(log);
+  const next: RunStatus = { ...s, outcome };
+  if (next.state === "passed" && outcome.notPassed) next.state = "failed";
+  try {
+    writeFileSync(statusPath(work, s.id), JSON.stringify(next, null, 2));
+  } catch {
+    // Read again next time: what lab_status answers is the same either way.
+  }
+  return next;
 }
 
 export function runDir(work: string, id: string): string {
@@ -300,16 +413,23 @@ export function readStatus(work: string, id: string): RunStatus | undefined {
   try {
     const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
     // A run whose process is gone while its status still says it is going was lost — the runtime or
-    // the machine restarted under it. Said, rather than left reading as running for ever.
-    if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
+    // the machine restarted under it. Said, rather than left reading as running for ever. One that never
+    // recorded a process (the runtime died between writing its status and starting it) is lost too: it
+    // would otherwise hold up every run after it (review 256).
+    if (!["passed", "failed", "stopped", "lost"].includes(s.state) && gone(s.pid)) {
       s.state = "lost";
     }
-    return s;
+    return settled(work, s);
   } catch {
     return undefined;
   }
 }
 
+/** gone is whether a job's process is not running: none recorded, or none by that id. */
+function gone(pid: number | undefined): boolean {
+  return !pid || !alive(pid);
+}
+
 function alive(pid: number): boolean {
   try {
     process.kill(pid, 0);
@@ -446,3 +566,238 @@ export function tail(work: string, id: string, lines: number): string {
     return "";
   }
 }
+
+// ---- the base image (novox/hq issue 698)
+//
+// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which
+// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the
+// package database the image was built with. On a rolling distribution the mirrors stop serving what an old
+// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror
+// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a
+// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image
+// older than the bound, and rebuilds the image when asked to.
+//
+// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is
+// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds
+// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little;
+// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test.
+
+/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */
+export const BASE_IMAGE_ALIAS = "mesh-lab/base";
+
+/** How old the base image may be, in days, before a run is refused (see the head of this file). */
+export const BASE_MAX_AGE_DAYS = 14;
+
+const DAY = 24 * 60 * 60 * 1000;
+
+export interface BaseImage {
+  alias: string;
+  /** Whether the image is there at all. */
+  exists: boolean;
+  /** When this machine made it, ISO. */
+  built?: string;
+  /** Its age in whole days. */
+  ageDays?: number;
+  /** Why its age could not be read, when it could not. */
+  problem?: string;
+}
+
+/**
+ * baseImageFrom reads `incus image list <alias> --format json`: the image whose aliases hold the base's
+ * name, and when it was made here (its upload, which a publish sets).
+ */
+export function baseImageFrom(listed: string | undefined, now: Date): BaseImage {
+  const alias = BASE_IMAGE_ALIAS;
+  if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" };
+  let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>;
+  try {
+    images = JSON.parse(listed);
+  } catch {
+    return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" };
+  }
+  const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias));
+  if (!image) return { alias, exists: false };
+  const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN))
+    .find((t) => Number.isFinite(t) && t > 0);
+  if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" };
+  return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) };
+}
+
+/** readBaseImage asks incus, on this machine, for the base image. */
+export function readBaseImage(now = new Date()): BaseImage {
+  const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" });
+  return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now);
+}
+
+/** The base image's age in one sentence, as lab_check and lab_run say it. */
+export function describeBase(b: BaseImage): string {
+  if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`;
+  if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`;
+  return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`;
+}
+
+/**
+ * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the
+ * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says
+ * so itself, and a bed that does not need it runs.
+ */
+export function staleBase(b: BaseImage): string | undefined {
+  if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined;
+  return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` +
+    `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` +
+    `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`;
+}
+
+/**
+ * baseProblem is what lab_check and lab_base say is wrong with the base image, or undefined: past the bound, not
+ * on this machine, or of an age that cannot be read (review 256). Only the first refuses a run.
+ */
+export function baseProblem(b: BaseImage): string | undefined {
+  if (b.problem || !b.exists) return describeBase(b);
+  return staleBase(b);
+}
+
+/**
+ * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which
+ * replaces the image a bed raises its machines from, or an image older than the bound.
+ */
+export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined {
+  if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`;
+  return staleBase(base);
+}
+
+// ---- its rebuild: mesh-lab's `base build`, in the background, as a run is.
+
+export interface BaseBuild {
+  id: string;
+  state: "building" | "built" | "failed" | "lost";
+  started: string;
+  ended?: string;
+  exit?: number;
+  pid?: number;
+}
+
+function buildStatusPath(work: string, id: string): string {
+  return join(work, id, "status.json");
+}
+
+/** The base builds, newest first. A build whose process is gone while it says building was lost. */
+export function listBaseBuilds(work: string): BaseBuild[] {
+  if (!existsSync(work)) return [];
+  const builds: BaseBuild[] = [];
+  for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) {
+    try {
+      const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild;
+      if (b.state === "building" && gone(b.pid)) b.state = "lost";
+      builds.push(b);
+    } catch {
+      // A directory without a status is not a build.
+    }
+  }
+  return builds.sort((a, b) => b.started.localeCompare(a.started));
+}
+
+/** The base build still going, if any. */
+export function buildingBase(work: string): BaseBuild | undefined {
+  return listBaseBuilds(work).find((b) => b.state === "building");
+}
+
+/** The last lines of a base build's log. */
+export function baseBuildLog(work: string, id: string, lines = 40): string {
+  try {
+    return readFileSync(join(work, id, "build.log"), "utf8").split("\n").slice(-Math.max(1, lines)).join("\n");
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it,
+ * then its `base build`, which replaces the image and reads back that it works before publishing it.
+ */
+export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string {
+  const lab = join(dir, "mesh-lab");
+  return `set -euo pipefail
+export GIT_TERMINAL_PROMPT=0
+${credentialGuard(credential)}
+cd ${shellQuote(dir)}
+git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
+${credential ? `rm -f ${shellQuote(credential)}` : ""}
+echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)"
+printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
+export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
+${installStep(lab, false)}
+cd ${shellQuote(lab)}
+node --experimental-strip-types src/cli.ts base build
+`;
+}
+
+/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */
+export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild {
+  const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`;
+  const dir = join(work, id);
+  mkdirSync(dir, { recursive: true });
+  const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
+  writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
+  // What a crashed or rebooted run or rebuild left behind goes before this one writes its own.
+  cleanCredentials(work);
+  const credentials = writeCredential(credentialPath(work, id), credential);
+  writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
+  const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
+node -e ${shellQuote(
+    `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
+  )} "$code"
+rm -f ${shellQuote(credentialPath(work, id))}
+rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
+  const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
+  child.unref();
+  build.pid = child.pid;
+  writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
+  return build;
+}
+
+// ---- starting a run or a rebuild: one job on the lab's machine at a time (review 256).
+//
+// **Everything that waits is done first; the check and the start are one step.** A run and a rebuild each
+// refused while the other was going, but each asked the package registry — an await — between its check and
+// writing its status, so a run and a rebuild asked together could both pass their checks and both start: a
+// bed raising machines from an image being replaced under it. Here the registry is asked first, and nothing
+// is awaited between reading what is going and writing the new job's status, so in the one runtime that
+// serves the lab's tools no other call runs in between.
+
+export interface Launch {
+  /** The address a run or a rebuild names the package registry by, asked of the forge (npmRegistry). */
+  registry: () => Promise<{ registry: string; problem?: string }>;
+  /** The base image as it is now. */
+  base: () => BaseImage;
+}
+
+export type Launched<T> = { started: true; registry: string; problem?: string; base: BaseImage; job: T }
+  | { started: false; reason: string; base: BaseImage };
+
+/** launchRun starts a run unless a run or a rebuild is going, or the base image is past its bound. */
+export async function launchRun(work: string, launch: Launch, begin: (registry: string) => RunStatus):
+  Promise<Launched<RunStatus>> {
+  const registry = await launch.registry();
+  // From here to begin(): nothing awaited.
+  const base = launch.base();
+  const busy = running(work);
+  if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, base };
+  const refused = runRefusal(base, buildingBase(work));
+  if (refused) return { started: false, reason: refused, base };
+  return { started: true, ...registry, base, job: begin(registry.registry) };
+}
+
+/** launchRebuild starts a rebuild of the base image unless a run or another rebuild is going. */
+export async function launchRebuild(work: string, launch: Launch, begin: (registry: string) => BaseBuild):
+  Promise<Launched<BaseBuild>> {
+  const registry = await launch.registry();
+  // From here to begin(): nothing awaited.
+  const base = launch.base();
+  const rebuilding = buildingBase(work);
+  if (rebuilding) return { started: false, reason: `${rebuilding.id} is still rebuilding it`, base };
+  // A run raises its machines from the image; replacing it under one would change what the run tests.
+  const busy = running(work);
+  if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image`, base };
+  return { started: true, ...registry, base, job: begin(registry.registry) };
+}

modules/mesh-knowledge/cmd/mesh-knowledge/manifest_test.go

@@ -121,6 +121,12 @@ func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) {
 		!reflect.DeepEqual(m.Tools, []string{HealthTool, HighestTool}) {
 		t.Fatalf("watch %v, tools %v", watch, m.Tools)
 	}
+	// The watch's verdict is whether the database serves, so its check names the provision it exercises: while the
+	// database's provider is unhealthy, this finding is held under the provider's condition (hq ADR 0240 rule 5,
+	// issue 145).
+	if h["needs"] != "postgres-database" {
+		t.Fatalf("watch's check needs %v; it exercises postgres-database", h["needs"])
+	}
 	if watch["env"].(map[string]any)["MESH_KNOWLEDGE_STATUS"] != m.Build.Artifacts[0].Env["MESH_KNOWLEDGE_STATUS"] {
 		t.Fatal("watch writes another status file than the health tool reads")
 	}

modules/mesh-knowledge/module.json

@@ -156,6 +156,7 @@
       "health": {
         "kind": "tool",
         "tool": "knowledge_health",
+        "needs": "postgres-database",
         "interval": "60s",
         "timeout": "10s",
         "looks": 2,

modules/n8n/module.json

@@ -97,6 +97,13 @@
     {
       "id": "server",
       "type": "container",
+      "health": {
+        "kind": "http",
+        "endpoint": "web",
+        "path": "/healthz/readiness",
+        "status": 200,
+        "needs": "postgres-database"
+      },
       "name": "n8n",
       "artifact": "server",
       "network": "n8n",

modules/nextcloud/module.json

@@ -83,6 +83,14 @@
     {
       "id": "server",
       "type": "container",
+      "health": {
+        "kind": "http",
+        "endpoint": "web",
+        "path": "/status.php",
+        "status": 200,
+        "body": "\"installed\":true",
+        "needs": "postgres-database"
+      },
       "name": "nextcloud",
       "image": "nextcloud@sha256:fb966733647ea03f0446b0c22eac9733c8eb616d37b960caca9d4c3010e14a08",
       "env-file": [

modules/umami/module.json

@@ -89,6 +89,15 @@
     {
       "id": "server",
       "type": "container",
+      "health": {
+        "kind": "http",
+        "endpoint": "web",
+        "path": "/api/share/mesh-health",
+        "status": 404,
+        "body": "\"code\":\"not-found\"",
+        "timeout": "10s",
+        "needs": "postgres-database"
+      },
       "name": "umami",
       "image": "ghcr.io/umami-software/umami@sha256:85909afc45bdcda1917394594a087421fdbb05610fded0fa9f6fb861abb2f367",
       "network": "umami",

predefined.json

@@ -1,13 +0,0 @@
-{
-  "agent-managed-settings": {
-    "type": "object",
-    "description": "The managed settings every agent session on a node is held to (Claude Code's managed-settings.json), as a JSON object.",
-    "trusted": true
-  },
-  "font-size": {
-    "type": "number",
-    "description": "The size a desktop program shows its text in, in points.",
-    "minimum": 6,
-    "maximum": 72
-  }
-}