A scheduled step may hold its module's own containers still (hq ADR 0189)

while-stopped names resource ids of the same module's containers; the host
stops them before the run and starts them again after it, in reverse order,
whatever the step did. The restart is deferred before the first stop and runs
on its own context, because the one real risk of this field is a window that
never closes.

Scheduled steps only: at apply the declaration is applied in order and a
run-once step already gates what follows.

examples/foundation-first-node-nats.lock

@@ -161,7 +161,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.mesh-controller.tool.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

internal/apply/apply.go

@@ -247,6 +247,20 @@ func ApplyKeeping(
 	// orphan is removed, and if a removal then fails the guard is already up. A stale opening on an
 	// adopted node is removed as any orphan is.
 	var protecting, orphans []store.Applied
+	// **What a declared process replaces is handed over, not removed first** (novox/hq issue 213).
+	// Every other orphan goes before anything is applied; one a process names under `replaces` is
+	// kept until that process is applied and still running a moment later, so whatever it was —
+	// the controller's container — answers until its replacement does, and goes on answering if
+	// the replacement never comes up.
+	replacedBy := map[string]string{}
+	for _, r := range d.Resources {
+		if p, ok := r.(*declaration.Process); ok {
+			for _, id := range p.Replaces {
+				replacedBy[id] = p.ID
+			}
+		}
+	}
+	handover := map[string][]store.Applied{}
 	for _, orphan := range known.Orphans(declared, origin) {
 		if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
 			protecting = append(protecting, orphan)
@@ -264,6 +278,10 @@ func ApplyKeeping(
 			log(fmt.Sprintf("  kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
 			continue
 		}
+		if by, replaced := replacedBy[orphan.ID]; replaced {
+			handover[by] = append(handover[by], orphan)
+			continue
+		}
 		orphans = append(orphans, orphan)
 	}
 	ordered := d.Resources
@@ -517,6 +535,11 @@ func ApplyKeeping(
 			if c, ok := resource.(*declaration.Container); ok && c.RunOnce {
 				gates = true
 			}
+			// And a run-once process, which is the same step hosted as a unit: a version whose
+			// preparation did not complete must not be started (novox/hq ADR 0135, issue 213).
+			if p, ok := resource.(*declaration.Process); ok && p.RunOnce {
+				gates = true
+			}
 			if gates {
 				failed.Gated = true
 				// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
@@ -607,6 +630,28 @@ func ApplyKeeping(
 			}
 			log(line)
 		}
+
+		// Its replacement applied: what it replaces goes now, once it is seen running (issue 213).
+		if waiting, has := handover[resource.Identity()]; has {
+			delete(handover, resource.Identity())
+			if err := handOver(ctx, run, resource, waiting, removeOrphan, &report, log); err != nil {
+				failures = append(failures, err)
+			}
+		}
+	}
+
+	// A replacement that did not apply — failed, skipped behind its module's step, held — leaves
+	// what it replaces running and recorded, said, for the next apply to hand over.
+	unhanded := make([]string, 0, len(handover))
+	for by := range handover {
+		unhanded = append(unhanded, by)
+	}
+	sort.Strings(unhanded)
+	for _, by := range unhanded {
+		for _, orphan := range handover[by] {
+			keptForReplacement(&report, log, orphan,
+				fmt.Sprintf("kept: %s, which replaces it, did not apply", by))
+		}
 	}
 
 	if !orphansRemoved {
@@ -1041,6 +1086,38 @@ type unitReloader interface {
 	ReloadUnits(ctx context.Context, run system.Runner) error
 }
 
+// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
+// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
+var serviceSettle = 2 * time.Second
+
+// stayedRunning is the state of a unit the host has just asked to run, read twice.
+//
+// **Because the first read races the failure.** A service manager returns when it has started the
+// process, and the unit is "activating" or "active" at that instant whatever the process is about
+// to do. A daemon that reads its configuration, refuses it and exits does so a fraction of a second
+// later — fail2ban took 221 milliseconds the day this was written — so a single read back says
+// running about a machine whose daemon is already gone, and the apply reports "restarted" for a
+// service that is dead. Every ban on both public machines was lost that way while every check
+// passed (novox/hq ADR 0184), which is the one shape of failure this host exists to refuse.
+//
+// So it looks again, after the moment in which that happens. It does not wait for a slow unit to
+// finish starting: a unit still coming up reads as running both times and is accepted, as before.
+// What this catches is a unit that was running and is not any more.
+func stayedRunning(ctx context.Context, sys system.System, run Runner, unit string) (string, error) {
+	state, err := sys.ServiceState(ctx, run, unit)
+	if err != nil || state != "running" {
+		return state, err
+	}
+	timer := time.NewTimer(serviceSettle)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return state, ctx.Err()
+	case <-timer.C:
+	}
+	return sys.ServiceState(ctx, run, unit)
+}
+
 func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
 	changed map[string]bool, previous store.Applied) (Outcome, error) {
 	if r.Stateless() {
@@ -1120,6 +1197,9 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 		// Read back. A service manager accepting a command says the transaction was accepted,
 		// not that the unit is running — one that starts and immediately dies satisfies it.
 		after, err := sys.ServiceState(ctx, run, r.Unit)
+		if err == nil && r.State == "running" {
+			after, err = stayedRunning(ctx, sys, run, r.Unit)
+		}
 		if err != nil {
 			return out, err
 		}
@@ -1140,7 +1220,7 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 		}
 		// Read back, for the same reason as above: a unit that starts and immediately dies
 		// satisfies a service manager and nothing else.
-		after, err := sys.ServiceState(ctx, run, r.Unit)
+		after, err := stayedRunning(ctx, sys, run, r.Unit)
 		if err != nil {
 			return out, err
 		}
@@ -1230,7 +1310,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service,
 	}
 	// Read back: it was running, and a restart or reload that left it otherwise is a failure —
 	// the machine's network manager down is not a change to report and move past.
-	after, err := sys.ServiceState(ctx, run, r.Unit)
+	after, err := stayedRunning(ctx, sys, run, r.Unit)
 	if err != nil {
 		return out, err
 	}
@@ -1378,6 +1458,15 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
 // undo is never reported as done; not fatal for a former target, which was never dropped by anyone.
 var errNoRemoval = errors.New("no way to remove")
 
+// exited is a command that ran and exited non-zero: its words, and the exit itself.
+type exited struct {
+	words string
+	exit  *exec.ExitError
+}
+
+func (e *exited) Error() string { return e.words }
+func (e *exited) Unwrap() error { return e.exit }
+
 // ExecRunner runs a real command, with stdin closed and output captured.
 func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
 	cmd := exec.CommandContext(ctx, name, args...)
@@ -1386,8 +1475,12 @@ func ExecRunner(ctx context.Context, name string, args ...string) (string, error
 	if err != nil {
 		var exit *exec.ExitError
 		if errors.As(err, &exit) {
-			return string(out), fmt.Errorf("%s exited %d: %s",
+			// The words as they always were, and the exit underneath them, so a caller asks the
-				name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
+			// code (system.ExitCode) rather than matching text that this line is free to reword.
+			return string(out), &exited{
+				words: fmt.Sprintf("%s exited %d: %s", name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))),
+				exit:  exit,
+			}
 		}
 		return string(out), fmt.Errorf("%s: %w", name, err)
 	}
@@ -1407,6 +1500,25 @@ func applyPackage(ctx context.Context, sys system.System, r *declaration.Package
 	if err != nil {
 		return out, err
 	}
+	if r.Absent {
+		// Declared absent (novox/hq ADR 0180): removed when it is here, left alone when it is not.
+		if !installed {
+			out.Action = "unchanged"
+			out.Detail = "not installed, as declared"
+			return out, nil
+		}
+		if err := sys.RemovePackage(ctx, run, r.Package); err != nil {
+			return out, fmt.Errorf("removing %s: %w", r.Package, err)
+		}
+		if still, err := sys.PackageInstalled(ctx, run, r.Package); err != nil {
+			return out, err
+		} else if still {
+			return out, fmt.Errorf("%s was removed without error and the package database still has it", r.Package)
+		}
+		out.Action = "removed"
+		out.Detail = "declared absent; its configuration is left where the package manager leaves it"
+		return out, nil
+	}
 	if installed {
 		out.Action = "unchanged"
 		out.Detail = "already installed"
@@ -1588,6 +1700,10 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, c := range r.Capabilities {
 		b.WriteString("cap " + c + "\n")
 	}
+	// And where it logs (ADR 0179): the runtime cannot move a running container's output.
+	if r.Logging != "" {
+		b.WriteString("log " + r.Logging + "\n")
+	}
 	// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
 	// moves and the install is reported "updated" and re-established. Added only when present, so no
 	// ordinary container's or run-once step's digest moves for a field it does not set.
@@ -1785,6 +1901,11 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 	for _, c := range r.Capabilities {
 		args = append(args, "--cap-add", c)
 	}
+	if r.Logging != "" {
+		// The journal keeps the container's name on every line (CONTAINER_NAME), which is what a
+		// jail matches on (novox/hq ADR 0179); `docker logs` keeps working against the journal.
+		args = append(args, "--log-driver", r.Logging)
+	}
 	for _, d := range r.Dns {
 		args = append(args, "--dns", d)
 	}
@@ -2390,3 +2511,102 @@ func moduleOf(identity string) (string, bool) {
 	}
 	return identity[:at], true
 }
+
+// handOver removes what a process replaces, once the process is running and still is a moment later
+// (novox/hq issue 213, ADR 0184). A replacement that is not up keeps what it replaces in place
+// and recorded, and is this apply's failure: the old one answers until a later apply finds the new
+// one up.
+func handOver(ctx context.Context, run Runner, resource declaration.Resource,
+	waiting []store.Applied, removeOrphan func(store.Applied) error, report *Report,
+	log func(string)) *Error {
+	p, ok := resource.(*declaration.Process)
+	if !ok {
+		return nil
+	}
+	if why := stillUp(ctx, run, p.Name+".service"); why != "" {
+		for _, orphan := range waiting {
+			keptForReplacement(report, log, orphan,
+				fmt.Sprintf("kept: %s, which replaces it, is not running (%s)", p.ID, why))
+		}
+		return &Error{Resource: p.ID, Err: fmt.Errorf(
+			"%s was started and is not running a moment later (%s), so what it replaces was kept: %s",
+			p.Name, why, appliedIDs(waiting)), Done: *report}
+	}
+	for _, orphan := range waiting {
+		if err := removeOrphan(orphan); err != nil {
+			var failed *Error
+			if errors.As(err, &failed) {
+				return failed
+			}
+			return &Error{Resource: orphan.ID, Err: err, Done: *report}
+		}
+	}
+	return nil
+}
+
+// handoverSettle is how long a replacement must stay up before what it replaces goes. Longer than a
+// service's second look (serviceSettle): this one decides whether the last thing answering is
+// removed, and a process that fails on its store or its bus does so after it opened them, not in the
+// first instant. A test sets it to nothing.
+var handoverSettle = 10 * time.Second
+
+// stillUp says why a process's unit is not up and staying up, or nothing when it is: active and
+// running at two looks handoverSettle apart, the same main process both times, restarted by
+// nothing in between. Stricter than stayedRunning, which reads "activating" as running — the state
+// a crash-looping unit is in while it waits to be started again, which is exactly the replacement
+// that must not be handed anything.
+func stillUp(ctx context.Context, run Runner, unit string) string {
+	look := func() (map[string]string, string) {
+		out, err := run(ctx, "systemctl", "show", unit, "--property=ActiveState", "--property=SubState",
+			"--property=MainPID", "--property=NRestarts")
+		if err != nil {
+			return nil, err.Error()
+		}
+		got := map[string]string{}
+		for _, line := range strings.Split(out, "\n") {
+			if key, value, found := strings.Cut(strings.TrimSpace(line), "="); found {
+				got[key] = value
+			}
+		}
+		if got["ActiveState"] != "active" || got["SubState"] != "running" {
+			return got, fmt.Sprintf("%s/%s", got["ActiveState"], got["SubState"])
+		}
+		return got, ""
+	}
+	first, why := look()
+	if why != "" {
+		return why
+	}
+	timer := time.NewTimer(handoverSettle)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err().Error()
+	case <-timer.C:
+	}
+	second, why := look()
+	if why != "" {
+		return why
+	}
+	if first["MainPID"] != second["MainPID"] || first["NRestarts"] != second["NRestarts"] {
+		return fmt.Sprintf("restarted while it was watched (pid %s → %s, restarts %s → %s)",
+			first["MainPID"], second["MainPID"], first["NRestarts"], second["NRestarts"])
+	}
+	return ""
+}
+
+// keptForReplacement reports an orphan kept because what replaces it is not yet in its place.
+func keptForReplacement(report *Report, log func(string), orphan store.Applied, detail string) {
+	report.Outcomes = append(report.Outcomes, Outcome{
+		ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "kept", Detail: detail,
+	})
+	log(fmt.Sprintf("  kept %s (%s): %s", orphan.ID, orphan.Target, strings.TrimPrefix(detail, "kept: ")))
+}
+
+func appliedIDs(applied []store.Applied) string {
+	ids := make([]string, 0, len(applied))
+	for _, a := range applied {
+		ids = append(ids, a.ID)
+	}
+	return strings.Join(ids, ", ")
+}

internal/apply/archive.go

@@ -66,16 +66,10 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
 		}
 	}
 
-	if err := os.MkdirAll(r.Path, 0o755); err != nil {
+	written, err := replaceWith(body, r.Path, r.Owner)
-		return out, err
-	}
-	written, err := unpack(body, r.Path)
 	if err != nil {
 		return out, err
 	}
-	if err := ownAll(r.Path, r.Owner); err != nil {
-		return out, err
-	}
 	out.Action = "updated"
 	if previous.Wrote == "" {
 		out.Action = "created"
@@ -84,6 +78,63 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
 	return out, nil
 }
 
+// replaceWith makes the directory exactly the archive (novox/hq issue 220).
+//
+// **The tree on disk is the archive and nothing else.** The digest is the whole identity of what
+// is unpacked here, so a file the previous archive had and this one does not must go. Unpacked over
+// the old tree, it stayed: a bundle rebuilt as one file per entrypoint kept the package directory
+// of the version before, which code could still import, and a fix that removed a file worked on a
+// fresh machine only. So the archive is unpacked into a fresh directory beside the old one, owned,
+// and swapped in by rename. A running process keeps the files it has open, and the old tree is
+// removed only once the new one is in place. A failed unpack leaves the old tree untouched.
+func replaceWith(body []byte, path, owner string) (int, error) {
+	parent := filepath.Dir(path)
+	if err := os.MkdirAll(parent, 0o755); err != nil {
+		return 0, err
+	}
+	fresh := path + ".unpacking"
+	replaced := path + ".replaced"
+	// What an interrupted earlier attempt left beside the directory.
+	for _, leftover := range []string{fresh, replaced} {
+		if err := os.RemoveAll(leftover); err != nil {
+			return 0, err
+		}
+	}
+	if err := os.Mkdir(fresh, 0o755); err != nil {
+		return 0, err
+	}
+	written, err := unpack(body, fresh)
+	if err == nil {
+		err = ownAll(fresh, owner)
+	}
+	if err != nil {
+		os.RemoveAll(fresh)
+		return written, err
+	}
+	hadOne := true
+	if err := os.Rename(path, replaced); err != nil {
+		if !os.IsNotExist(err) {
+			os.RemoveAll(fresh)
+			return written, err
+		}
+		hadOne = false
+	}
+	if err := os.Rename(fresh, path); err != nil {
+		if hadOne {
+			// Put the old tree back rather than leave nothing at the path.
+			os.Rename(replaced, path)
+		}
+		os.RemoveAll(fresh)
+		return written, err
+	}
+	if hadOne {
+		if err := os.RemoveAll(replaced); err != nil {
+			return written, fmt.Errorf("%s is in place, and the tree it replaced could not be removed: %w", path, err)
+		}
+	}
+	return written, nil
+}
+
 func fetch(ctx context.Context, source string) ([]byte, error) {
 	request, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
 	if err != nil {

internal/apply/archive_replace_test.go

@@ -0,0 +1,68 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// novox/hq issue 220: the tree on disk is exactly the archive. A file the previous archive had and
+// this one does not is gone, and nothing is left beside the directory.
+func TestAnArchiveReplacesTheTreeItWasUnpackedOver(t *testing.T) {
+	dir := t.TempDir()
+	first, firstDigest := anArchive(t, map[string]string{"index.js": "old", "node_modules/dep/index.js": "dep"})
+	d := declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, first)+
+		`","digest":"`+firstDigest+`","path":"`+dir+`/tools"}`)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	second, secondDigest := anArchive(t, map[string]string{"index.js": "one file"})
+	d = declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, second)+
+		`","digest":"`+secondDigest+`","path":"`+dir+`/tools"}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "one file" {
+		t.Fatalf("index.js is %q", got)
+	}
+	if _, err := os.Stat(dir + "/tools/node_modules"); err == nil {
+		t.Fatal("the previous archive's directory is still there")
+	}
+	entries, _ := os.ReadDir(dir)
+	if len(entries) != 1 {
+		names := []string{}
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		t.Fatalf("beside the tree: %v", names)
+	}
+}
+
+// A tree is replaced only by one that unpacked whole: an archive refused halfway leaves the old
+// tree as it was.
+func TestARefusedArchiveLeavesTheTreeItWouldHaveReplaced(t *testing.T) {
+	dir := t.TempDir()
+	first, firstDigest := anArchive(t, map[string]string{"index.js": "old"})
+	d := declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, first)+
+		`","digest":"`+firstDigest+`","path":"`+dir+`/tools"}`)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bad, badDigest := anArchive(t, map[string]string{"index.js": "new", "../../escaped": "no"})
+	d = declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, bad)+
+		`","digest":"`+badDigest+`","path":"`+dir+`/tools"}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil); err == nil {
+		t.Fatal("an escaping archive was accepted")
+	}
+	if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "old" {
+		t.Fatalf("index.js is %q after a refused archive", got)
+	}
+	if entries, _ := os.ReadDir(dir); len(entries) != 1 {
+		t.Fatalf("%d entries beside the tree after a refused archive", len(entries))
+	}
+}

internal/apply/handover_test.go

@@ -0,0 +1,318 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// novox/hq issue 213: the controller moves from a container to a process on the one machine that
+// runs it. Every orphan is removed before anything is applied, so without a handover the container
+// went first and nothing answered the mesh's verbs while the process was fetched, unpacked and
+// started — and for ever, if it did not start. A process that `replaces` the container is applied
+// first; the container goes only once the process is running a moment later.
+
+// aMachine fakes the service manager and the container runtime: it records every command, answers
+// `systemctl show` with whether the process is running, and has a container until it is removed.
+type aMachine struct {
+	commands  []string
+	running   bool // what `systemctl show` says of the process once it was started
+	started   bool
+	container bool
+	timer     bool // whether a timer, once started, is up
+	crashing  bool // up at the first look, waiting to restart at the next
+	looks     int
+}
+
+func (m *aMachine) run(ctx context.Context, name string, args ...string) (string, error) {
+	line := name + " " + strings.Join(args, " ")
+	m.commands = append(m.commands, line)
+	switch {
+	case name == "systemctl" && len(args) > 0 && (args[0] == "restart" || args[0] == "start"):
+		m.started = true
+	case name == "systemctl" && len(args) > 0 && args[0] == "is-active" && strings.HasSuffix(args[len(args)-1], ".timer"):
+		if m.timer {
+			return "active", nil
+		}
+		return "inactive", errors.New("inactive")
+	case name == "systemctl" && len(args) > 0 && args[0] == "is-active":
+		if m.started && m.running {
+			return "active", nil
+		}
+		return "inactive", errors.New("inactive")
+	case name == "systemctl" && len(args) > 0 && args[0] == "show":
+		m.looks++
+		if m.crashing && m.started {
+			// Up at the first look; waiting to be started again, a new process, at the second.
+			if m.looks == 1 {
+				return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
+			}
+			return "ActiveState=activating\nSubState=auto-restart\nMainPID=0\nNRestarts=1\n", nil
+		}
+		if m.started && m.running {
+			return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
+		}
+		return "ActiveState=inactive\nSubState=dead\nMainPID=0\nNRestarts=0\n", nil
+	case name == "docker" && len(args) > 1 && args[0] == "rm":
+		m.container = false
+	case name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect":
+		if !m.container {
+			return "", errors.New("no such container")
+		}
+		return "true\t", nil
+	}
+	return "", nil
+}
+
+func (m *aMachine) index(prefix string) int {
+	for i, c := range m.commands {
+		if strings.HasPrefix(c, prefix) {
+			return i
+		}
+	}
+	return -1
+}
+
+// theController is a machine whose controller ran as a container, recorded, and a declaration that
+// runs it as a process from a bundle served here instead.
+func theController(t *testing.T, digest, source, extra string) (store.State, string) {
+	t.Helper()
+	known := store.State{Resources: []store.Applied{{
+		Origin: store.OriginDeclared, ID: "mesh-controller.server", Type: "container", Target: "mesh-controller",
+	}}}
+	return known, `{"declaration":1,"resources":[
+	  {"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + source +
+		`","digest":"` + digest + `","run":["./mesh-controller","serve"]` + extra + `}]}`
+}
+
+func onAMachine(t *testing.T) {
+	t.Helper()
+	serviceSettle, handoverSettle = 0, 0
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = t.TempDir(), t.TempDir()
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+}
+
+func TestAContainerAProcessReplacesGoesOnlyOnceTheProcessRuns(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{running: true, container: true}
+
+	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err != nil {
+		t.Fatalf("the handover failed: %v", err)
+	}
+	started, removed := m.index("systemctl restart mesh-controller.service"), m.index("docker rm -f mesh-controller")
+	if started < 0 || removed < 0 {
+		t.Fatalf("the process was not started or the container not removed: %v", m.commands)
+	}
+	if removed < started {
+		t.Fatalf("the container was removed before its replacement was started — a window with "+
+			"nothing answering: %v", m.commands)
+	}
+	if looked := m.index("systemctl show mesh-controller.service"); looked < 0 || looked > removed {
+		t.Errorf("the container was removed without looking whether the process runs: %v", m.commands)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
+		t.Errorf("the container's outcome is %+v, want removed", o)
+	}
+	if _, still := after.Find("mesh-controller.server"); still {
+		t.Error("the host still records the container it removed")
+	}
+	if _, has := after.Find("mesh-controller.controller"); !has {
+		t.Error("the process was not recorded")
+	}
+}
+
+// The case the handover exists for: the replacement does not stay up. The container keeps
+// answering, stays recorded so a later apply hands it over, and the apply says why it failed.
+func TestAContainerIsKeptWhenItsReplacementDoesNotRun(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{running: false, container: true}
+
+	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err == nil {
+		t.Fatal("a replacement that is not running was reported as a clean apply")
+	}
+	if !strings.Contains(err.Error(), "mesh-controller.server") {
+		t.Errorf("the failure does not name what was kept: %v", err)
+	}
+	if m.index("docker rm") >= 0 {
+		t.Fatalf("the container was removed though its replacement is not running: %v", m.commands)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
+		t.Errorf("the container's outcome is %+v, want kept", o)
+	}
+	if _, still := after.Find("mesh-controller.server"); !still {
+		t.Fatal("the container was forgotten, so no later apply would ever remove it")
+	}
+
+	// The next apply finds the process up and finishes the handover.
+	m.running, m.commands = true, nil
+	report, after, err = Apply(context.Background(), archHost(t), parse(t, raw), after,
+		store.OriginDeclared, m.run, nil, nil)
+	if err != nil {
+		t.Fatalf("the second apply failed: %v", err)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
+		t.Errorf("the second apply did not hand over: %+v (%v)", o, m.commands)
+	}
+	if _, still := after.Find("mesh-controller.server"); still {
+		t.Error("the container is still recorded after the handover")
+	}
+}
+
+// A replacement that never applied — its bundle is not what was declared — touches nothing, and
+// what it replaces keeps running.
+func TestAContainerIsKeptWhenItsReplacementFailsToApply(t *testing.T) {
+	onAMachine(t)
+	body, _ := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, "sha256:"+strings.Repeat("b", 64), serving(t, body),
+		`,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{running: true, container: true}
+
+	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err == nil {
+		t.Fatal("a replacement whose bundle did not match was reported applied")
+	}
+	if m.index("docker rm") >= 0 {
+		t.Fatalf("the container was removed though nothing replaced it: %v", m.commands)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
+		t.Errorf("the container's outcome is %+v, want kept", o)
+	}
+	if _, still := after.Find("mesh-controller.server"); !still {
+		t.Fatal("the container was forgotten")
+	}
+}
+
+// Without `replaces` nothing changes: an orphan goes before anything is applied, as it always has.
+// Kept as a test because it is the window the field exists to close.
+func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), ``)
+	m := &aMachine{running: true, container: true}
+	if _, _, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if removed, started := m.index("docker rm -f mesh-controller"), m.index("systemctl restart mesh-controller.service"); removed < 0 || removed > started {
+		t.Fatalf("an orphan nothing replaces was not removed first: %v", m.commands)
+	}
+}
+
+// novox/hq issue 213, beyond what the oneshot unit (process_step_test.go) already holds: a step
+// written `./name` runs its own bundle's binary — the controller's preparation is its own binary —
+// and a step is started, never enabled.
+func TestAStepRunsItsOwnBundlesBinaryAndIsNotEnabled(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	m := &aMachine{running: true}
+	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
+	  "source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil); err != nil {
+		t.Fatalf("the step failed: %v", err)
+	}
+	for _, c := range m.commands {
+		if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") {
+			t.Errorf("the step was run directly or enabled: %v", m.commands)
+		}
+	}
+	unit, err := os.ReadFile(filepath.Join(unitDir, "mesh-controller-prepare.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare"
+	if !strings.Contains(string(unit), want) {
+		t.Errorf("the step does not run its own bundle's binary (%q):\n%s", want, unit)
+	}
+}
+
+func TestAStepThatFailsGatesItsModule(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	src := serving(t, body)
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if name == "systemctl" && len(args) > 1 && args[0] == "start" {
+			return "", errors.New("Job for mesh-controller-prepare.service failed")
+		}
+		if name == "systemctl" && len(args) > 0 && args[0] == "restart" {
+			t.Errorf("the module's process was started after its step failed")
+		}
+		return "", nil
+	}
+	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
+	  "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true},
+	  {"id":"mesh-controller.controller","type":"process","name":"mesh-controller",
+	  "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","serve"]}`)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
+	if err == nil {
+		t.Fatal("a failed step was reported as a clean apply")
+	}
+	if o := outcomeOf(report, "mesh-controller.controller"); o.Action != "skipped" {
+		t.Errorf("the process after a failed step was %+v, want skipped", o)
+	}
+}
+
+// A completed step is done: applied again unchanged, it is not run again — its service is never up
+// between runs, and reading that as "a daemon that stopped" re-ran the controller's preparation on
+// every apply. Nor is a scheduled run started off its cadence; its timer is what is kept up.
+func TestACompletedStepIsNotRunAgainAndAScheduleIsItsTimer(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"job": "#!/bin/sh\n"})
+	src := serving(t, body)
+	for _, mode := range []string{`"run-once":true`, `"schedule":"0 3 * * *"`} {
+		// The service of either is never up between runs; a scheduled one's timer is.
+		m := &aMachine{running: false, timer: true}
+		d := declare(t, `{"id":"m.job","type":"process","name":"m-job","source":"`+src+`","digest":"`+digest+
+			`","run":["./job"],`+mode+`}`)
+		_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
+		if err != nil {
+			t.Fatalf("%s: first apply: %v", mode, err)
+		}
+		m.commands = nil
+		report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, nil)
+		if err != nil {
+			t.Fatalf("%s: second apply: %v", mode, err)
+		}
+		if m.index("systemctl start m-job.service") >= 0 || m.index("systemctl restart m-job.service") >= 0 {
+			t.Errorf("%s: an unchanged apply ran the job again: %v", mode, m.commands)
+		}
+		if o := outcomeOf(report, "m.job"); o.Action != "unchanged" {
+			t.Errorf("%s: an unchanged apply reported %+v", mode, o)
+		}
+	}
+}
+
+// A replacement crash-looping between its restarts is not up, though the service manager calls it
+// "activating" — the state the host's ordinary second look accepts as running. Removing the
+// container on that reading would leave nothing answering.
+func TestAContainerIsKeptWhenItsReplacementIsCrashLooping(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{crashing: true, container: true}
+	_, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err == nil || !strings.Contains(err.Error(), "auto-restart") {
+		t.Fatalf("a crash-looping replacement was accepted: %v", err)
+	}
+	if m.index("docker rm") >= 0 {
+		t.Fatalf("the container was removed for a replacement that keeps dying: %v", m.commands)
+	}
+	if _, still := after.Find("mesh-controller.server"); !still {
+		t.Fatal("the container was forgotten")
+	}
+}

internal/apply/left_out_test.go

@@ -173,3 +173,42 @@ func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
 		t.Fatal("a capability is not part of the container's spec")
 	}
 }
+
+// A package may be declared absent (novox/hq ADR 0180): removed when it is installed, read back,
+// left alone when it is not.
+func TestAPackageDeclaredAbsentIsRemovedWhenPresentAndLeftWhenNot(t *testing.T) {
+	installed := true
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if name != "pacman" {
+			return "", nil
+		}
+		switch args[0] {
+		case "-Q":
+			if args[1] == "pacman" || installed {
+				return args[1] + " 1.0\n", nil
+			}
+			return "", errors.New("package not found")
+		case "-R":
+			installed = false
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[{"id":"front-end","type":"package","package":"ufw","absent":true}]}`)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "removed" || !strings.Contains(strings.Join(ran, "\n"), "pacman -R --noconfirm ufw") {
+		t.Fatalf("an installed package declared absent was not removed: %+v\n%v", report.Outcomes[0], ran)
+	}
+	ran = nil
+	report, _, err = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "unchanged" || strings.Contains(strings.Join(ran, "\n"), "-R") {
+		t.Fatalf("a package already absent was touched: %+v\n%v", report.Outcomes[0], ran)
+	}
+}

internal/apply/logging_test.go

@@ -0,0 +1,43 @@
+package apply
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A container declared to log to the journal is run with the journal as its log driver, and the
+// place it logs is part of its spec, so moving it recreates the container (novox/hq ADR 0179).
+func TestAContainerLoggingToTheJournalIsRunThatWayAndRecreatedWhenMoved(t *testing.T) {
+	pinned := "postgres@sha256:" + strings.Repeat("a", 64)
+	var ran []string
+	run := func(_ context.Context, cmd string, args ...string) (string, error) {
+		if cmd == "docker" && len(args) > 0 && args[0] == "run" {
+			ran = args
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"front","type":"container","name":"front","image":"`+pinned+`","logging":"journald"}
+	]}`)
+	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	sent := false
+	for i, a := range ran {
+		if a == "--log-driver" && i+1 < len(ran) && ran[i+1] == "journald" {
+			sent = true
+		}
+	}
+	if !sent {
+		t.Fatalf("the container's output was not sent to the journal: %v", ran)
+	}
+	with := d.Resources[0].(*declaration.Container)
+	without := *with
+	without.Logging = ""
+	if containerSpec(with, inputs{}) == containerSpec(&without, inputs{}) {
+		t.Fatal("where a container logs is not part of its spec, so moving it would not recreate it")
+	}
+}

internal/apply/maintenance_window_test.go

@@ -0,0 +1,196 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// A scheduled step may hold its module's own containers still while it runs (novox/hq ADR 0189,
+// issue 108).
+//
+// What it exists for: the artifact store's collector walks the storage and requires every writer
+// stopped. A run-once step runs beside containers and a scheduled one is the same container again,
+// so the mesh had no way to say it — which is why the store it inherited has never collected
+// anything. The risk the field brings is one shape only: a window that opens and never closes.
+// Every test here is about that shape.
+
+// windowRun records the order of stop / run / start, which is the whole of what is being asserted.
+type windowRun struct {
+	mu     sync.Mutex
+	order  []string
+	failAt string // the arg[0] that should fail ("run" makes the step fail)
+	wontGo string // a container name that refuses to start again
+}
+
+func (w *windowRun) run(_ context.Context, _ string, args ...string) (string, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	switch args[0] {
+	case "info":
+		return "27.0\n", nil
+	case "stop", "start":
+		w.order = append(w.order, args[0]+" "+args[1])
+		if args[0] == "start" && args[1] == w.wontGo {
+			return "", errors.New("the runtime refused")
+		}
+	case "run":
+		w.order = append(w.order, "run")
+		if w.failAt == "run" {
+			return "", errors.New("the step exited non-zero")
+		}
+	}
+	return "", nil
+}
+
+func (w *windowRun) seen() []string {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return append([]string{}, w.order...)
+}
+
+// aStoreWithACollector is a module in the shape distribution has: a server that must not be
+// writing, and a nightly step that walks its storage with the server held still.
+func aStoreWithACollector(t *testing.T) *declaration.Declaration {
+	t.Helper()
+	return parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"store","type":"container","name":"mesh-registry","image":"`+pinned+`"},
+	  {"id":"collect","type":"container","name":"mesh-registry-collect","image":"`+pinned+`",
+	   "schedule":"30 3 * * *","while-stopped":["store"]}
+	]}`)
+}
+
+func fireOnce(t *testing.T, d *declaration.Declaration, w *windowRun) {
+	t.Helper()
+	clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
+	s := NewScheduler(clock, w.run, func(string) {})
+	s.Sync(d, nil)
+	s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
+	s.Wait()
+}
+
+func TestAScheduledStepHoldsItsModulesContainerStillAndStartsItAgain(t *testing.T) {
+	w := &windowRun{}
+	fireOnce(t, aStoreWithACollector(t), w)
+
+	got := w.seen()
+	want := []string{"stop mesh-registry", "run", "start mesh-registry"}
+	var kept []string
+	for _, line := range got {
+		if strings.HasPrefix(line, "stop mesh-registry-collect") {
+			// Clearing the step's own exited container by name; not part of the window.
+			continue
+		}
+		kept = append(kept, line)
+	}
+	if len(kept) != len(want) {
+		t.Fatalf("the window was not stop, run, start: %v", got)
+	}
+	for i := range want {
+		if kept[i] != want[i] {
+			t.Fatalf("the window was %v, want %v", kept, want)
+		}
+	}
+}
+
+// The one that matters: a step that fails must leave the service running.
+func TestAFailedStepStillClosesTheWindow(t *testing.T) {
+	w := &windowRun{failAt: "run"}
+	fireOnce(t, aStoreWithACollector(t), w)
+
+	var started bool
+	for _, line := range w.seen() {
+		if line == "start mesh-registry" {
+			started = true
+		}
+	}
+	if !started {
+		t.Fatalf("the step failed and the container it held still was never started again: %v", w.seen())
+	}
+}
+
+// A container that will not come back is said loudly: it is down, and nothing else notices until
+// the next apply compares it.
+func TestAContainerThatWillNotStartAgainIsSaidLoudly(t *testing.T) {
+	w := &windowRun{wontGo: "mesh-registry"}
+	var said []string
+	clock := &fixedClock{now: time.Date(2026, 10, 2, 3, 29, 0, 0, time.UTC)}
+	s := NewScheduler(clock, w.run, func(line string) { said = append(said, line) })
+	s.Sync(aStoreWithACollector(t), nil)
+	s.Advance(context.Background(), time.Date(2026, 10, 2, 3, 30, 5, 0, time.UTC))
+	s.Wait()
+
+	var loud bool
+	for _, line := range said {
+		if strings.Contains(line, "WILL NOT START AGAIN") && strings.Contains(line, "mesh-registry") {
+			loud = true
+		}
+	}
+	if !loud {
+		t.Fatalf("a service left stopped by a maintenance window was not said loudly: %v", said)
+	}
+}
+
+// Several containers come back in the reverse of the order they were stopped: a module names the
+// dependant first, and starting it before what it depends on is not bringing it back.
+func TestTheWindowClosesInTheReverseOfTheOrderItOpened(t *testing.T) {
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"web","type":"container","name":"web","image":"`+pinned+`"},
+	  {"id":"db","type":"container","name":"db","image":"`+pinned+`"},
+	  {"id":"collect","type":"container","name":"collect","image":"`+pinned+`",
+	   "schedule":"30 3 * * *","while-stopped":["web","db"]}
+	]}`)
+	w := &windowRun{}
+	fireOnce(t, d, w)
+
+	var stops, starts []string
+	for _, line := range w.seen() {
+		switch {
+		case line == "stop web" || line == "stop db":
+			stops = append(stops, line)
+		case strings.HasPrefix(line, "start "):
+			starts = append(starts, line)
+		}
+	}
+	if len(stops) != 2 || stops[0] != "stop web" || stops[1] != "stop db" {
+		t.Fatalf("stopped in %v, want the order the step named them", stops)
+	}
+	if len(starts) != 2 || starts[0] != "start db" || starts[1] != "start web" {
+		t.Fatalf("started in %v, want the reverse", starts)
+	}
+}
+
+// And the refusals, each for what it says rather than that it says something.
+func TestAMaintenanceWindowIsRefusedWhereItCannotMean(t *testing.T) {
+	for _, c := range []struct{ name, body, says string }{
+		{
+			"a window with no schedule",
+			`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","while-stopped":["store"]}`,
+			"needs a schedule",
+		},
+		{
+			"a window naming itself",
+			`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["collect"]}`,
+			"this step itself",
+		},
+		{
+			"a window naming something that is not a container here",
+			`{"id":"collect","type":"container","name":"c","image":"` + pinned + `","schedule":"30 3 * * *","while-stopped":["elsewhere"]}`,
+			"no container by that id",
+		},
+	} {
+		_, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` + c.body + `]}`))
+		if err == nil {
+			t.Errorf("%s was accepted", c.name)
+			continue
+		}
+		if !strings.Contains(err.Error(), c.says) {
+			t.Errorf("%s: the refusal does not say %q: %v", c.name, c.says, err)
+		}
+	}
+}

internal/apply/opening.go

@@ -76,6 +76,16 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri
 	if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive {
 		return "", nil
 	}
+	if !firewall.Installed(ctx, run) {
+		// Uninstalled (novox/hq ADR 0180): retired for good, by the module that replaced it. Said
+		// once, and nothing is asked of a command that is not there.
+		if rec.RetiredBy != firewall.RetiredRemoved {
+			rec.RetiredBy = firewall.RetiredRemoved
+			log("  the found firewall (ufw) is no longer installed; the mesh's filter is what filters this machine")
+			return "removed: ufw is no longer installed; the mesh's filter is what filters this machine", nil
+		}
+		return "", nil
+	}
 	active := firewall.Active(ctx, run)
 	if !active && !(rec.Forward != nil && !rec.DisabledByMesh) {
 		// Inactive, and either the mesh's doing already or nobody's recorded here: said as found,

internal/apply/opening_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
@@ -522,3 +523,33 @@ func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
 		t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall)
 	}
 }
+
+// A front end that is no longer installed is recorded as removed, said once, and asked nothing of
+// (novox/hq ADR 0180).
+func TestAnUninstalledFrontEndIsRetiredForGood(t *testing.T) {
+	dir := t.TempDir()
+	u := &ufwMachine{installed: false, ruleset: "table inet mesh\n"}
+	known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true,
+		RetiredBy: "mesh", FoundAt: time.Now()}}
+	converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
+	report, state, err := applyWith(t, converged, known, u.run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state.Firewall.RetiredBy != "removed" || !strings.Contains(report.Firewall, "no longer installed") {
+		t.Fatalf("record %+v, said %q", state.Firewall, report.Firewall)
+	}
+	u.asked = nil
+	report, _, err = applyWith(t, converged, state, u.run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Firewall != "" {
+		t.Errorf("said again: %q", report.Firewall)
+	}
+	for _, a := range u.asked {
+		if strings.HasPrefix(a, "ufw") && a != "ufw status" {
+			t.Errorf("asked something of a front end that is not there: %v", u.asked)
+		}
+	}
+}

internal/apply/process.go

@@ -77,11 +77,26 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		// Everything about it is as declared. Still asked whether it is RUNNING, because a
 		// declaration that is satisfied by a record rather than by the machine is how a stopped
 		// service reports success.
-		if active, err := run(ctx, "systemctl", "is-active", "--quiet", r.Name+".service"); err == nil {
+		// **The record is carried forward, not re-derived.** An unchanged outcome is recorded
+		// like any other, so one that said nothing about what was written erased the digest; the
+		// next apply then found no record, re-created the daemon, and the one after that found a
+		// record again — the node's runtime restarted every other cycle (novox/hq 04-ISSUES/210).
+		out.wrote = want
+		// **A step that ran is done, and a schedule is its timer** (novox/hq issue 213). Neither's
+		// service is meant to be up between runs, so asking whether it is — and starting it when it
+		// was not — ran a completed step again on every apply, and a scheduled run off its cadence.
+		if r.RunOnce {
+			return out, nil
+		}
+		unit := r.Name + ".service"
+		if r.Schedule != "" {
+			unit = r.Name + ".timer"
+		}
+		if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
 			_ = active
 			return out, nil
 		}
-		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
+		if _, err := run(ctx, "systemctl", "start", unit); err != nil {
 			return out, fmt.Errorf("%s is installed and would not start: %w", r.Name, err)
 		}
 		out.Action = "updated"
@@ -110,9 +125,22 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 	// so the machine is not asked to start something that needed a migration that did not happen.
 	// Nothing is left behind to ask afterwards: the record that it ran is the digest, which is why
 	// the identity above includes the command.
+	//
+	// **Run as the unit a daemon would be, once** (novox/hq design 38 WP4c). Run directly, the
+	// step started in the host's own working directory, without its environment, its environment
+	// files or its user — `node bootstrap/index.js` resolved from wherever the host ran and was told
+	// none of the words it was declared with. A oneshot unit carries all four exactly as a daemon's
+	// does, and starting one waits for it to finish and fails when it fails.
 	if r.RunOnce {
-		if _, err := run(ctx, r.Run[0], r.Run[1:]...); err != nil {
+		unit := filepath.Join(unitDir, r.Name+".service")
-			return out, fmt.Errorf("the %s step did not complete: %w", r.Name, err)
+		if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
+			return out, err
+		}
+		if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
+			return out, err
+		}
+		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
+			return out, fmt.Errorf("the %s step did not complete (journalctl -u %s.service says why): %w", r.Name, r.Name, err)
 		}
 		out.Action = "created"
 		if previous.Wrote != "" {
@@ -209,9 +237,9 @@ func unitFor(r *declaration.Process) string {
 	if r.User != "" {
 		fmt.Fprintf(&b, "User=%s\n", r.User)
 	}
-	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(r.Run, " "))
+	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
-	if r.Schedule != "" {
+	if r.Schedule != "" || r.RunOnce {
-		// Started by its timer and expected to finish. Restarting it would have it run
+		// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
 		// continuously between fires, which is the opposite of a schedule.
 		b.WriteString("Type=oneshot\n")
 		b.WriteString("\n")
@@ -341,3 +369,15 @@ func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, st
 	}
 	return "removed", "stopped; its unit and its bundle removed — the mesh's own code", nil
 }
+
+// runFrom is the command as the unit runs it. **A command written `./name` is that file in the
+// process's own unpacked bundle** (novox/hq ADR 0193): a bundle compiled to a binary runs itself,
+// and only the host knows where it unpacked it, while the service manager takes an absolute path or
+// a name it finds on its own search path — never one relative to the working directory.
+func runFrom(r *declaration.Process) []string {
+	run := append([]string(nil), r.Run...)
+	if len(run) > 0 && strings.HasPrefix(run[0], "./") {
+		run[0] = filepath.Join(daemonRoot, r.Name, strings.TrimPrefix(run[0], "./"))
+	}
+	return run
+}

internal/apply/process_step_test.go

@@ -0,0 +1,81 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A run-once process is a step run where, how and as whom it was declared (novox/hq design 38
+// WP4c): its bundle's directory, its environment and environment files, its user. Run directly,
+// it started in the host's own directory with none of them.
+func TestARunOnceProcessRunsAsItsOneshotUnit(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+
+	me, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, digest := anArchive(t, map[string]string{"bootstrap/index.js": "console.log(1)\n"})
+	var commands []string
+	fail := false
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		commands = append(commands, name+" "+strings.Join(args, " "))
+		if fail && name == "systemctl" && len(args) > 0 && args[0] == "start" {
+			return "", errors.New("exit status 1")
+		}
+		return "", nil
+	}
+	d := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"user":"`+me.Username+`",`+
+		`"env":{"MESH_ADMIN":"mesh-admin"},"env-file":["/var/lib/mesh/mosquitto/bootstrap.env"]}`)
+
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	unit, err := os.ReadFile(filepath.Join(units, "mosquitto-bootstrap.service"))
+	if err != nil {
+		t.Fatalf("no unit was written for the step: %v", err)
+	}
+	for _, want := range []string{
+		"WorkingDirectory=" + filepath.Join(bundles, "mosquitto-bootstrap"),
+		"EnvironmentFile=/var/lib/mesh/mosquitto/bootstrap.env",
+		`Environment="MESH_ADMIN=mesh-admin"`,
+		"User=" + me.Username,
+		"Type=oneshot",
+		"ExecStart=node bootstrap/index.js",
+	} {
+		if !strings.Contains(string(unit), want) {
+			t.Errorf("the step's unit lacks %q:\n%s", want, unit)
+		}
+	}
+	for _, never := range []string{"Restart=always", "[Install]", "Type=simple"} {
+		if strings.Contains(string(unit), never) {
+			t.Errorf("a step's unit says %q:\n%s", never, unit)
+		}
+	}
+	joined := strings.Join(commands, "; ")
+	if !strings.Contains(joined, "systemctl start mosquitto-bootstrap.service") {
+		t.Errorf("the step was not started as its unit: %s", joined)
+	}
+	if strings.Contains(joined, "node bootstrap/index.js") || strings.Contains(joined, "enable mosquitto-bootstrap") {
+		t.Errorf("the step was run directly or enabled: %s", joined)
+	}
+
+	// A step that fails fails the apply, and is not recorded as done.
+	fail = true
+	d2 := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"env":{"MESH_ADMIN":"changed"}}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, run, nil, nil); err == nil {
+		t.Error("a step that failed did not fail the apply")
+	}
+}

internal/apply/process_test.go

@@ -256,3 +256,73 @@ func TestAProcessRecordedUnderAPathlikeNameIsRefusedNotRemoved(t *testing.T) {
 		}
 	}
 }
+
+// novox/hq 04-ISSUES/210: the node's runtime was re-created — and restarted — on every reconcile,
+// because the host did not find what it wrote for a process the cycle before. Applying the same
+// process declaration twice must do no work the second time.
+func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+
+	body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"})
+	var commands []string
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		commands = append(commands, name+" "+strings.Join(args, " "))
+		return "", nil
+	}
+	d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
+
+	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
+		t.Fatalf("first apply: %+v, want created", o)
+	}
+	rec, ok := state.Find("node-tools.runtime")
+	if !ok || rec.Wrote == "" {
+		t.Fatalf("the host did not record what it wrote for the process: %+v", rec)
+	}
+	commands = nil
+	again, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(again, "node-tools.runtime"); o.Action != "unchanged" {
+		t.Errorf("second apply: %+v, want unchanged", o)
+	}
+	for _, c := range commands {
+		if strings.Contains(c, "restart") {
+			t.Errorf("the second apply restarted the process: %v", commands)
+		}
+	}
+	// And the record survives an unchanged apply: the third cycle is unchanged too. This is the
+	// cycle the live mesh showed — created, unchanged, created — before the record was carried.
+	if rec, _ := state.Find("node-tools.runtime"); rec.Wrote == "" {
+		t.Fatalf("an unchanged apply dropped the digest from the record: %+v", rec)
+	}
+	commands = nil
+	third, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(third, "node-tools.runtime"); o.Action != "unchanged" {
+		t.Errorf("third apply: %+v, want unchanged", o)
+	}
+}
+
+// novox/hq ADR 0193: a bundle compiled to a binary runs itself — `./name` is that file in the
+// process's own unpacked bundle, made absolute because the service manager takes nothing relative.
+func TestAProcessRunsItsOwnBundlesBinary(t *testing.T) {
+	unit := unitFor(&declaration.Process{Name: "node-tools", Run: []string{"./node-tools", "serve"}})
+	if !strings.Contains(unit, "ExecStart="+filepath.Join(daemonRoot, "node-tools", "node-tools")+" serve\n") {
+		t.Errorf("the binary is not run from its bundle:\n%s", unit)
+	}
+	other := unitFor(&declaration.Process{Name: "x", Run: []string{"node", "src/main.js"}})
+	if !strings.Contains(other, "ExecStart=node src/main.js\n") {
+		t.Errorf("a command found on the path was changed:\n%s", other)
+	}
+}

internal/apply/schedule.go

@@ -65,6 +65,29 @@ type scheduledJob struct {
 	container *declaration.Container
 	next      time.Time // the next minute at which it is due
 	running   bool      // a run is in flight — the next due run is skipped rather than stacked
+	// hold is the runtime names of the containers held still for the duration of a run, in the
+	// order the step named them (novox/hq ADR 0189).
+	hold []string
+}
+
+// heldStillFor is the runtime names of the containers a step holds still, resolved from ids.
+func heldStillFor(step *declaration.Container, d *declaration.Declaration) []string {
+	if len(step.WhileStopped) == 0 {
+		return nil
+	}
+	byID := map[string]string{}
+	for _, r := range d.Resources {
+		if c, ok := r.(*declaration.Container); ok {
+			byID[c.Identity()] = c.Name
+		}
+	}
+	out := make([]string, 0, len(step.WhileStopped))
+	for _, id := range step.WhileStopped {
+		if name := byID[id]; name != "" {
+			out = append(out, name)
+		}
+	}
+	return out
 }
 
 // NewScheduler builds a scheduler. A nil clock is the system clock; a nil log says nothing.
@@ -123,15 +146,20 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
 		// on its cadence rather than staying running to be restarted — so its identity cannot
 		// depend on another resource's content and there is nothing to pass.
 		spec := containerSpec(c, inputs{})
+		// The runtime stops containers by name; the declaration names them by id. Resolved here,
+		// against the declaration this job was armed from, so a fire never has to look anything up
+		// (novox/hq ADR 0189). The parser has already refused an id that is not a container here.
+		hold := heldStillFor(c, d)
 		if existing := s.jobs[c.Identity()]; existing != nil && existing.spec == spec {
 			// Unchanged: keep where it is in its cadence, refresh the declaration pointer only.
 			existing.container = c
+			existing.hold = hold
 			continue
 		}
 		// New or changed: arm it for the next due minute after now.
 		next, _ := cron.Next(s.clock.Now())
 		s.jobs[c.Identity()] = &scheduledJob{
-			id: c.Identity(), spec: spec, cron: cron, container: c, next: next,
+			id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold,
 		}
 	}
 
@@ -202,6 +230,15 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) {
 		return
 	}
 
+	// **The window opens here and closes in the defer, whatever happens** (novox/hq ADR 0189).
+	// Deferred before the first stop so a panic, a failing step or a step that runs long all end
+	// the same way: the service running. The one real risk of this field is a window that never
+	// closes, and the only defence against it is that closing is not conditional on anything.
+	if len(j.hold) > 0 {
+		defer s.letRun(ctx, cri, j)
+		s.holdStill(ctx, cri, j)
+	}
+
 	// A container by this name left exited by the previous run would collide with --name. Removing
 	// one that is not there is the state we want, so its error is ignored — the same as run-once.
 	_, _ = s.run(ctx, cri, "rm", "-f", j.container.Name)
@@ -260,3 +297,51 @@ func (s *Scheduler) Run(ctx context.Context) {
 		}
 	}
 }
+
+// holdStill stops the containers this step runs instead of, in the order it named them.
+//
+// A stop that fails is said and not fatal. The step runs anyway: for the case this exists for —
+// a collector walking storage nothing must be writing to — a writer that would not stop is worth
+// knowing about, and refusing to run would mean the work never happens and the log says nothing
+// new each night. What must not be skipped is the restart, and it is not: it is deferred.
+func (s *Scheduler) holdStill(ctx context.Context, cri string, j *scheduledJob) {
+	for _, name := range j.hold {
+		if _, err := s.run(ctx, cri, "stop", name); err != nil {
+			s.log(fmt.Sprintf("scheduled step %s: could not stop %s for the run: %v", j.id, name, err))
+			continue
+		}
+		s.log(fmt.Sprintf("scheduled step %s: %s held still for the run", j.id, name))
+	}
+}
+
+// letRun starts them again, in the reverse of the order they were stopped, and says so loudly if
+// one does not come back.
+//
+// **Reverse order**, because stopping walks a dependency the other way: a module that holds two
+// containers still names the one that depends on the other first, and bringing them back the same
+// way would start a dependant before what it depends on.
+//
+// Given its own context, because this runs in a defer and the one the run used may already be
+// cancelled — a host shutting down mid-window would otherwise leave the service stopped, which is
+// precisely the outcome this field must never have.
+func (s *Scheduler) letRun(_ context.Context, cri string, j *scheduledJob) {
+	ctx, cancel := context.WithTimeout(context.Background(), closingWindow)
+	defer cancel()
+	for i := len(j.hold) - 1; i >= 0; i-- {
+		name := j.hold[i]
+		if _, err := s.run(ctx, cri, "start", name); err != nil {
+			// Said as loudly as this host says anything: a service the mesh stopped for a
+			// maintenance window and could not start again is down, and nothing else will notice
+			// until the next apply compares it.
+			s.log(fmt.Sprintf(
+				"scheduled step %s: %s was held still for the run and WILL NOT START AGAIN: %v",
+				j.id, name, err))
+			continue
+		}
+		s.log(fmt.Sprintf("scheduled step %s: %s running again", j.id, name))
+	}
+}
+
+// closingWindow is how long the host will spend putting back what it stopped. Generous: this is
+// the half that must not be given up on.
+const closingWindow = 5 * time.Minute

internal/apply/stayed_running_test.go

@@ -0,0 +1,100 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A daemon that reads a configuration it refuses dies a fraction of a second after the service
+// manager has reported it started — fail2ban took 221 milliseconds on the control node the day this
+// was written. One read back catches nothing: the unit is active at that instant. The mesh reported
+// the service "restarted" while every ban on two public machines was gone, and every check passed
+// (novox/hq ADR 0184). The host looks again, after the moment in which that happens.
+func TestAServiceThatDiesJustAfterItsRestartIsNotReportedRestarted(t *testing.T) {
+	serviceSettle = 0
+	// Alive at the first look after starting, dead at the second — the shape of a daemon that
+	// refuses what it was just given.
+	started, looks := false, 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			state := "active"
+			if started {
+				if looks++; looks >= 2 {
+					state = "failed"
+				}
+			}
+			return "LoadState=loaded\nActiveState=" + state + "\n", nil
+		}
+		if args[0] == "start" {
+			started = true
+		}
+		return "", nil
+	}
+	dir := t.TempDir()
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"conf","type":"file","path":"`+dir+`/jail.conf","content":"[sshd]\n","mode":"0644"},
+	  {"id":"run","type":"service","unit":"fail2ban.service","state":"running","restart-on":["conf"]}
+	]}`)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err == nil {
+		t.Fatal("a service that died just after being restarted was reported as restarted")
+	}
+	if !strings.Contains(err.Error(), "fail2ban.service") || !strings.Contains(err.Error(), "is stopped") {
+		t.Errorf("the failure does not name the unit and what it is now: %v", err)
+	}
+	if looks < 2 {
+		t.Errorf("the host looked at the unit %d time(s) after starting it; it must look again", looks)
+	}
+}
+
+// A unit still coming up reads as running at both looks and is accepted: the second look is for a
+// unit that WAS running and is not any more, never a wait for a slow one to finish starting.
+func TestAUnitStillStartingIsNotAFailure(t *testing.T) {
+	serviceSettle = 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			return "LoadState=loaded\nActiveState=activating\n", nil
+		}
+		return "", nil
+	}
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"s","type":"service","unit":"slow.service","state":"running"}
+	]}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("a unit still starting was reported as a failure: %v", err)
+	}
+}
+
+// And a service the declaration asks to be stopped is not waited on at all.
+func TestAServiceAskedToStopIsNotWaitedOn(t *testing.T) {
+	serviceSettle = 0
+	shows := 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			shows++
+			if shows == 1 {
+				return "LoadState=loaded\nActiveState=active\n", nil
+			}
+			return "LoadState=loaded\nActiveState=inactive\n", nil
+		}
+		return "", nil
+	}
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"s","type":"service","unit":"off.service","state":"stopped"}
+	]}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("stopping a service was reported as a failure: %v", err)
+	}
+}
+
+// The settle between a unit's two read-backs is a real pause on a machine and nothing in a test:
+// no test here drives a service manager that takes time, so paying it would only slow the suite
+// (novox/hq ADR 0184).
+func TestMain(m *testing.M) {
+	serviceSettle = 0
+	os.Exit(m.Run())
+}

internal/apply/testing_dirs.go

@@ -0,0 +1,13 @@
+package apply
+
+// ForTests points where the host writes units and unpacks daemons at directories a test owns, and
+// waits nothing between its looks at a unit, until the returned function puts them back. For tests
+// in other packages that apply a process — the bootstrap's, which checks that what genesis raises is
+// what the controller's process takes over (novox/hq issue 223). Nothing outside a test calls it.
+func ForTests(units, daemons string) (restore func()) {
+	wasUnits, wasDaemons, wasSettle, wasHandover := unitDir, daemonRoot, serviceSettle, handoverSettle
+	unitDir, daemonRoot, serviceSettle, handoverSettle = units, daemons, 0, 0
+	return func() {
+		unitDir, daemonRoot, serviceSettle, handoverSettle = wasUnits, wasDaemons, wasSettle, wasHandover
+	}
+}

internal/bootstrap/build.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -112,6 +114,43 @@ func BuildControlPlane(ctx context.Context, run Runner, builderTag string, sourc
 		return Built{}, nil
 	}
 
+	// **Which form the controller is in at that commit** (novox/hq issue 223). An image the module
+	// builds is the form genesis always raised, and the builder builds it as before. A process the
+	// module runs from a Go bundle cannot be built here — its toolchain is one the mesh makes later —
+	// so genesis builds the controller's own Dockerfile and raises it as the container that process
+	// replaces (genesis_form.go).
+	workspace, err := os.MkdirTemp("", "mesh-genesis-*")
+	if err != nil {
+		return Built{}, err
+	}
+	defer os.RemoveAll(workspace)
+	dir, commit, err := cloneAt(ctx, run, builderTag, source, workspace)
+	if err != nil {
+		return Built{}, fmt.Errorf("the control plane could not be fetched from %s at %s: %w",
+			source.Repository, shortRef(source.Ref), err)
+	}
+	raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
+	if err != nil {
+		return Built{}, fmt.Errorf("%s at %s has no module manifest: %w", source.Repository, shortRef(source.Ref), err)
+	}
+	form, err := processFormOf(raw)
+	if err != nil {
+		return Built{}, err
+	}
+	if form.Found {
+		image, err := buildGenesisImage(ctx, run, dir)
+		if err != nil {
+			return Built{}, err
+		}
+		manifest, err := genesisForm(raw, form, image)
+		if err != nil {
+			return Built{}, err
+		}
+		say(fmt.Sprintf("  built %s from %s, as the container its process %s replaces (%s)",
+			ControlPlaneModule, shortRef(commit), form.Process, form.Replaces))
+		return Built{Module: ControlPlaneModule, Commit: commit, Image: image, Manifest: manifest}, nil
+	}
+
 	out, err := run(ctx, "docker", args...)
 	if err != nil {
 		return Built{}, fmt.Errorf("the control plane could not be built from %s at %s: %w",

internal/bootstrap/build_test.go

@@ -49,10 +49,10 @@ func TestARepositoryAndACommitIsEnough(t *testing.T) {
 func TestTheBuildHandsOverTheManifestTheMeshWillHold(t *testing.T) {
 	manifest := `{"module":"mesh-controller","version":"1","resources":[` +
 		`{"id":"server","type":"container","name":"mesh-controller","image":"` + builtImage + `"}]}`
-	runtime := &asked{answer: func(string, []string) (string, error) {
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
 		return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":` + manifest +
 			`,"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}` + "\n", nil
-	}}
+	})}
 	built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
 		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
 	if err != nil {
@@ -69,10 +69,10 @@ func TestTheBuildHandsOverTheManifestTheMeshWillHold(t *testing.T) {
 // A result without a manifest is a build the installer cannot finish, and it is refused beside the
 // builder that said it rather than at step 9 with a message about a missing file.
 func TestABuildReportingNoManifestIsRefused(t *testing.T) {
-	runtime := &asked{answer: func(string, []string) (string, error) {
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
 		return `{"module":"mesh-controller","commit":"a1b2c3d4",` +
 			`"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
-	}}
+	})}
 	_, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
 		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
 	if err == nil || !strings.Contains(err.Error(), "no manifest") {
@@ -82,11 +82,11 @@ func TestABuildReportingNoManifestIsRefused(t *testing.T) {
 
 // And a manifest that does not name the image the build produced describes some other build.
 func TestABuildWhoseManifestNamesAnotherImageIsRefused(t *testing.T) {
-	runtime := &asked{answer: func(string, []string) (string, error) {
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
 		return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":{"module":"mesh-controller",` +
 			`"resources":[{"id":"server","type":"container","image":"sha256:` + strings.Repeat("9", 64) + `"}]},` +
 			`"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
-	}}
+	})}
 	_, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
 		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
 	if err == nil || !strings.Contains(err.Error(), "does not name that image") {

internal/bootstrap/genesis_form.go

@@ -0,0 +1,205 @@
+package bootstrap
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// The controller as a process, raised at genesis as a container (novox/hq issue 213, issue 223).
+//
+// **The mesh runs the controller as a Go bundle the host starts as a process; genesis cannot.** A
+// process's bundle is fetched from the mesh's artifact store, which genesis raises long after the
+// controller, and a Go bundle is compiled in a toolchain the mesh builds later still. So genesis
+// pivots to the controller as it always has — an image it built from the controller's own
+// Dockerfile, run as a container the temporary controller composes — and the first time the mesh
+// builds the controller from its repository, the controller's own declaration is the process, which
+// names that container under `replaces`, and the host hands over: the process is started, seen up,
+// and only then is the container removed (mesh-host `replaces`, issue 213).
+//
+// **The container is genesis's shape, not the manifest's.** The manifest declares only the process.
+// From it genesis takes what the process is given — its environment, which is host paths and words —
+// and the id the process replaces; the container around it is written here: the image genesis built,
+// the host's network, and every host path the environment names mounted at the same path read-only.
+// It runs as the image's own unprivileged user (65534), so the secrets belong to that number until
+// the process's account takes them over — the image is FROM scratch and knows no account by name. Its
+// id is the one the process replaces, so the first declaration the mesh composes for this machine
+// hands this container over rather than leaving two controllers running.
+
+// genesisUser is who the genesis container runs as — the image's own USER — and who its secrets
+// belong to until the process takes them over: the image has no passwd to look an account up in.
+const genesisUser = "65534:65534"
+
+// ProcessForm is the controller's process, as its manifest declares it, and the container id that
+// process replaces. Found is false for a manifest in the image form — an older controller — which
+// genesis installs as it always did.
+type ProcessForm struct {
+	Found    bool
+	Process  string // the process resource's id
+	Replaces string // the id of the container genesis raises in its place
+}
+
+// processFormOf finds the controller's process in its manifest: a process resource running a bundle
+// the module builds, saying which one resource it replaces.
+func processFormOf(manifest []byte) (ProcessForm, error) {
+	var m struct {
+		Build *struct {
+			Artifacts []struct {
+				Name string `json:"name"`
+				Kind string `json:"kind"`
+			} `json:"artifacts"`
+		} `json:"build"`
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(manifest, &m); err != nil {
+		return ProcessForm{}, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err)
+	}
+	kinds := map[string]string{}
+	if m.Build != nil {
+		for _, a := range m.Build.Artifacts {
+			kinds[a.Name] = a.Kind
+		}
+	}
+	for _, kind := range kinds {
+		if kind == "image" {
+			return ProcessForm{}, nil // the image form: the builder builds it, as before
+		}
+	}
+	var found []ProcessForm
+	for _, r := range m.Resources {
+		if r["type"] != "process" || kinds[fmt.Sprint(r["artifact"])] != "bundle" {
+			continue
+		}
+		if once, _ := r["run-once"].(bool); once {
+			continue
+		}
+		replaces, _ := r["replaces"].([]any)
+		if len(replaces) != 1 {
+			return ProcessForm{}, fmt.Errorf(
+				"the %s module runs as the process %v and says it replaces %v. Genesis raises the "+
+					"controller as a container that process takes over, so the process names exactly "+
+					"one resource it replaces — the id genesis gives the container",
+				ControlPlaneModule, r["id"], r["replaces"])
+		}
+		found = append(found, ProcessForm{Found: true, Process: fmt.Sprint(r["id"]),
+			Replaces: fmt.Sprint(replaces[0])})
+	}
+	if len(found) != 1 {
+		return ProcessForm{}, fmt.Errorf(
+			"the %s module builds no image and runs %d process(es) of its own; genesis raises one "+
+				"controller, from the process its manifest declares", ControlPlaneModule, len(found))
+	}
+	return found[0], nil
+}
+
+// genesisForm is the manifest genesis registers: the controller's own manifest, its process
+// replaced by the container genesis runs in its place, under the id the process replaces, running
+// the image genesis built. Resolved as a build would resolve it — no build section, the image named
+// — because that is what the temporary controller is handed.
+func genesisForm(manifest []byte, form ProcessForm, image string) ([]byte, error) {
+	var m map[string]any
+	if err := json.Unmarshal(manifest, &m); err != nil {
+		return nil, err
+	}
+	resources, _ := m["resources"].([]any)
+	var out []any
+	for _, raw := range resources {
+		r, _ := raw.(map[string]any)
+		if r == nil || r["id"] != form.Process {
+			out = append(out, raw)
+			continue
+		}
+		env, _ := r["env"].(map[string]any)
+		var volumes []any
+		for _, key := range sortedAnyKeys(env) {
+			value := fmt.Sprint(env[key])
+			if strings.HasPrefix(value, "/") || strings.HasPrefix(value, "${dir:") {
+				volumes = append(volumes, value+":"+value+":ro")
+			}
+		}
+		container := map[string]any{
+			"id": form.Replaces, "type": "container", "name": ControlPlaneModule,
+			"image": image, "network": "host", "args": []any{"serve"},
+		}
+		if len(env) > 0 {
+			container["env"] = env
+		}
+		if len(volumes) > 0 {
+			container["volumes"] = volumes
+		}
+		out = append(out, container)
+	}
+	m["resources"] = out
+	// What the container reads must be readable by who it runs as. The process's account owns them
+	// once the process takes over, and the host gives them to it in the same apply.
+	m["secrets-owner"] = genesisUser
+	// **Nothing to prepare at genesis.** The temporary controller — the same commit — migrated the
+	// stores when the foundation raised it, and a preparation step is derived from a resource running
+	// an artifact the module built, which a pinned image is not: the controller refuses `prepares`
+	// with nothing to run it in. The process prepares the stores itself when it takes over.
+	delete(m, "prepares")
+	delete(m, "build")
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(m); err != nil {
+		return nil, err
+	}
+	return bytes.TrimSpace(b.Bytes()), nil
+}
+
+func sortedAnyKeys(m map[string]any) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// cloneAt fetches the controller's repository at the commit genesis builds, into a directory of
+// this machine's, with the carried builder's git — the machine is not assumed to have one. Returns
+// the module's directory and the commit that was checked out.
+func cloneAt(ctx context.Context, run Runner, builderTag string, source Source, into string) (string, string, error) {
+	git := func(args ...string) (string, error) {
+		return run(ctx, "docker", append([]string{"run", "--rm", "-v", into + ":/ws",
+			"--entrypoint", "git", builderTag}, args...)...)
+	}
+	if _, err := git("clone", "--quiet", source.Repository, "/ws/src"); err != nil {
+		return "", "", fmt.Errorf("cloning %s: %w", source.Repository, err)
+	}
+	if _, err := git("-C", "/ws/src", "checkout", "--quiet", "--detach", source.Ref); err != nil {
+		return "", "", fmt.Errorf("checking out %s: %w", shortRef(source.Ref), err)
+	}
+	commit, err := git("-C", "/ws/src", "rev-parse", "HEAD")
+	if err != nil {
+		return "", "", err
+	}
+	return filepath.Join(into, "src", source.Path), strings.TrimSpace(commit), nil
+}
+
+// buildGenesisImage builds the controller's image from its own Dockerfile (the one `make image`
+// uses), on this machine, and names it by the digest of its own configuration, as the builder did.
+func buildGenesisImage(ctx context.Context, run Runner, dir string) (string, error) {
+	if _, err := os.Stat(filepath.Join(dir, "Dockerfile")); err != nil {
+		return "", fmt.Errorf("the %s repository has no Dockerfile, so genesis has no image to raise "+
+			"the controller from: %w", ControlPlaneModule, err)
+	}
+	out, err := run(ctx, "docker", "build", "--quiet", dir)
+	if err != nil {
+		return "", fmt.Errorf("building the %s image: %w", ControlPlaneModule, err)
+	}
+	image := strings.TrimSpace(out)
+	if i := strings.LastIndex(image, "\n"); i >= 0 {
+		image = strings.TrimSpace(image[i+1:])
+	}
+	if !strings.HasPrefix(image, "sha256:") {
+		return "", fmt.Errorf("docker build said %q, which is not an image id", firstLine(out))
+	}
+	return image, nil
+}

internal/bootstrap/genesis_form_test.go

@@ -0,0 +1,182 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// imageFormManifest is a controller from before issue 213: it builds an image and runs a container.
+const imageFormManifest = `{"module":"mesh-controller","version":"1","resources":[
+  {"id":"server","type":"container","name":"mesh-controller","network":"host","artifact":"server"}],
+  "build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}}`
+
+// processFormManifest is the controller's manifest as novox/mesh-controller declares it after issue
+// 213: a Go bundle the host runs as a process, replacing the container it ran as.
+const processFormManifest = `{
+  "module": "mesh-controller", "version": "1", "slug": "control", "prepares": true,
+  "claims": [{"name": "mesh-controller", "scope": "mesh"}],
+  "accesses": [{"path": "/var/lib/mesh-broker-tls", "mode": "read"}],
+  "own-secrets": {"inventory": "${dir:mesh-state}/inventory", "bus": "${dir:mesh-state}/bus"},
+  "secrets-owner": "mesh-controller",
+  "tools": ["status"],
+  "resources": [
+    {"id": "account", "type": "user", "name": "mesh-controller", "shell": "/usr/bin/nologin", "home": "/var/lib/mesh-controller"},
+    {"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh", "owner": "mesh-controller"},
+    {"id": "controller", "type": "process", "name": "mesh-controller", "artifact": "controller",
+     "run": ["./mesh-controller", "serve"], "user": "mesh-controller",
+     "env": {"MESH_BROKER_CERTIFICATE": "/var/lib/mesh-broker-tls/tls.crt",
+             "MESH_STORE_INVENTORY_FILE": "${dir:mesh-state}/inventory",
+             "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
+             "MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"},
+     "replaces": ["server"]}
+  ],
+  "build": {"artifacts": [{"name": "controller", "kind": "bundle", "language": "go", "system": "arch",
+    "from": "cmd/mesh-controller", "binary": "mesh-controller"}]}
+}`
+
+// aRepository answers the carried builder's git as a clone of a repository holding this manifest and
+// a Dockerfile, and hands every other command to then.
+func aRepository(t *testing.T, manifest string, then func(string, []string) (string, error)) func(string, []string) (string, error) {
+	t.Helper()
+	return func(name string, args []string) (string, error) {
+		if name == "docker" && len(args) > 5 && args[0] == "run" && args[4] == "--entrypoint" && args[5] == "git" {
+			host := strings.TrimSuffix(args[3], ":/ws")
+			joined := strings.Join(args, " ")
+			switch {
+			case strings.Contains(joined, " clone "):
+				src := filepath.Join(host, "src")
+				if err := os.MkdirAll(src, 0o755); err != nil {
+					return "", err
+				}
+				if err := os.WriteFile(filepath.Join(src, "module.json"), []byte(manifest), 0o644); err != nil {
+					return "", err
+				}
+				return "", os.WriteFile(filepath.Join(src, "Dockerfile"), []byte("FROM scratch\n"), 0o644)
+			case strings.Contains(joined, "rev-parse"):
+				return "a1b2c3d4e5f6\n", nil
+			}
+			return "", nil
+		}
+		return then(name, args)
+	}
+}
+
+// novox/hq issue 223: a controller declared as a process is raised at genesis as a container built
+// from its own Dockerfile, not by the builder — which has no Go toolchain at genesis and would refuse.
+func TestAProcessFormControllerIsBuiltFromItsDockerfileAndRaisedAsAContainer(t *testing.T) {
+	runtime := &asked{answer: aRepository(t, processFormManifest, func(name string, args []string) (string, error) {
+		if name == "docker" && args[0] == "build" {
+			return builtImage + "\n", nil
+		}
+		t.Fatalf("genesis ran %s %v; a process-form controller is built from its Dockerfile alone", name, args)
+		return "", nil
+	})}
+	built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
+		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if built.Image != builtImage || built.Commit != "a1b2c3d4e5f6" {
+		t.Errorf("built %q from %q", built.Image, built.Commit)
+	}
+	if runtime.ran("mesh-builder:test build") {
+		t.Error("the builder was asked to build a controller it cannot build at genesis")
+	}
+
+	var m map[string]any
+	if err := json.Unmarshal(built.Manifest, &m); err != nil {
+		t.Fatalf("the genesis manifest is not JSON: %v", err)
+	}
+	if _, has := m["prepares"]; has {
+		t.Error("the genesis manifest prepares its state; the temporary controller already did, and a pinned image is nothing the controller can derive a step from")
+	}
+	if _, has := m["build"]; has {
+		t.Error("the genesis manifest still says how it is built; it is handed over resolved")
+	}
+	var container map[string]any
+	for _, raw := range m["resources"].([]any) {
+		r := raw.(map[string]any)
+		if r["type"] == "process" {
+			t.Errorf("the genesis manifest still runs the process: %v", r)
+		}
+		if r["type"] == "container" {
+			container = r
+		}
+	}
+	if container == nil {
+		t.Fatal("the genesis manifest runs no container")
+	}
+	for key, want := range map[string]any{"id": "server", "name": "mesh-controller", "image": builtImage,
+		"network": "host"} {
+		if container[key] != want {
+			t.Errorf("the genesis container's %s is %v, not %v", key, container[key], want)
+		}
+	}
+	if _, has := container["user"]; has {
+		t.Error("the genesis container says a user; the host's container has no such field, the image's USER is who it runs as")
+	}
+	if m["secrets-owner"] != "65534:65534" {
+		t.Errorf("the secrets belong to %v, which the container cannot read as", m["secrets-owner"])
+	}
+	volumes, _ := json.Marshal(container["volumes"])
+	for _, want := range []string{"${dir:mesh-state}/inventory:${dir:mesh-state}/inventory:ro",
+		"/var/lib/mesh-broker-tls/tls.crt:/var/lib/mesh-broker-tls/tls.crt:ro"} {
+		if !strings.Contains(string(volumes), want) {
+			t.Errorf("the container does not mount %s: %s", want, volumes)
+		}
+	}
+	if strings.Contains(string(volumes), "seat:") {
+		t.Errorf("a word that is not a path was mounted: %s", volumes)
+	}
+
+	// And it is what step 9 installs: pinned, its container found, its stores delivered from its
+	// environment — the same path an image-form controller takes.
+	pinned, _, err := pinImage(built.Manifest, built.Image, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id := controlPlaneResourceIn(pinned); id != "server" {
+		t.Errorf("step 9 finds the controller's container as %q", id)
+	}
+	wanted, err := secretsByVariableIn(pinned)
+	if err != nil {
+		t.Fatalf("step 9 cannot deliver the stores into the genesis container: %v", err)
+	}
+	if wanted["MESH_STORE_INVENTORY"] != "inventory" {
+		t.Errorf("step 9 delivers %v", wanted)
+	}
+}
+
+// An older controller — an image and a container — is still built by the builder, as before.
+func TestAnImageFormControllerIsStillBuiltByTheBuilder(t *testing.T) {
+	manifest := `{"module":"mesh-controller","version":"1","resources":[` +
+		`{"id":"server","type":"container","name":"mesh-controller","image":"` + builtImage + `"}]}`
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(name string, args []string) (string, error) {
+		if name == "docker" && args[0] == "build" {
+			t.Fatal("an image-form controller was built from its Dockerfile rather than by the builder")
+		}
+		return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":` + manifest +
+			`,"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
+	})}
+	built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
+		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(built.Manifest) != manifest || !runtime.ran("mesh-builder:test build") {
+		t.Errorf("the image form did not go through the builder: %s", built.Manifest)
+	}
+}
+
+func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) {
+	for _, bad := range []string{`"replaces": []`, `"replaces": ["a", "b"]`} {
+		raw := strings.Replace(processFormManifest, `"replaces": ["server"]`, bad, 1)
+		if _, err := processFormOf([]byte(raw)); err == nil {
+			t.Errorf("a process saying %s was accepted; genesis would not know what to name its container", bad)
+		}
+	}
+}

internal/bootstrap/genesis_handover_test.go

@@ -0,0 +1,121 @@
+package bootstrap
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// novox/hq issue 223: what genesis raises is what the controller's process takes over. The temporary
+// controller composes the genesis container, and the host records it as `<module>.<its id>`; the
+// first declaration the mesh composes from the controller's real manifest names that same id under
+// the process's `replaces` (the composer prefixes both alike — mesh-controller's
+// TestTheControllerIsAProcessAndNoContainer). So the first apply hands over: the process is started,
+// seen up, and only then is the genesis container removed — one controller before, one after, never
+// none and never two left.
+func TestTheFirstApplyHandsTheGenesisContainerOverToTheProcess(t *testing.T) {
+	restore := apply.ForTests(t.TempDir(), t.TempDir())
+	defer restore()
+
+	form, err := processFormOf([]byte(processFormManifest))
+	if err != nil {
+		t.Fatal(err)
+	}
+	genesis, err := genesisForm([]byte(processFormManifest), form, builtImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// What the host records for the genesis container, as the composer names it.
+	recorded := ""
+	var m struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(genesis, &m); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range m.Resources {
+		if r["type"] == "container" {
+			recorded = ControlPlaneModule + "." + r["id"].(string)
+		}
+	}
+	known := store.State{Resources: []store.Applied{{Origin: store.OriginDeclared, ID: recorded,
+		Type: "container", Target: ControlPlaneModule}}}
+
+	// The controller's first composed declaration: its process, replacing what the manifest names.
+	body, digest := aBundle(t)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }))
+	defer server.Close()
+	replaces, _ := json.Marshal([]string{ControlPlaneModule + "." + form.Replaces})
+	d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"` + ControlPlaneModule + "." + form.Process +
+		`","type":"process","name":"mesh-controller","source":"` + server.URL + `/c.tgz","digest":"` + digest +
+		`","run":["./mesh-controller","serve"],"replaces":` + string(replaces) + `}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var commands []string
+	started, container := false, true
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		line := name + " " + strings.Join(args, " ")
+		commands = append(commands, line)
+		switch {
+		case strings.HasPrefix(line, "systemctl restart mesh-controller.service"):
+			started = true
+		case strings.HasPrefix(line, "systemctl show mesh-controller.service") && started:
+			return "ActiveState=active\nSubState=running\nMainPID=7\nNRestarts=0\n", nil
+		case strings.HasPrefix(line, "docker rm -f mesh-controller"):
+			if !started {
+				t.Error("the genesis container was removed before the process was started")
+			}
+			container = false
+		case strings.HasPrefix(line, "docker container inspect") && !container:
+			return "", errors.New("no such container")
+		}
+		return "", nil
+	}
+	sys, err := system.For("arch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, after, err := apply.Apply(context.Background(), sys, d, known, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatalf("the first apply did not hand over: %v\n%s", err, strings.Join(commands, "\n"))
+	}
+	if container {
+		t.Fatalf("the genesis container is still running beside the process — two controllers:\n%s",
+			strings.Join(commands, "\n"))
+	}
+	if _, still := after.Find(recorded); still {
+		t.Error("the host still records the genesis container")
+	}
+}
+
+func aBundle(t *testing.T) ([]byte, string) {
+	t.Helper()
+	var raw bytes.Buffer
+	zipped := gzip.NewWriter(&raw)
+	w := tar.NewWriter(zipped)
+	content := "#!/bin/sh\n"
+	if err := w.WriteHeader(&tar.Header{Name: "mesh-controller", Mode: 0o755, Size: int64(len(content)), Typeflag: tar.TypeReg}); err != nil {
+		t.Fatal(err)
+	}
+	_, _ = w.Write([]byte(content))
+	_ = w.Close()
+	_ = zipped.Close()
+	sum := sha256.Sum256(raw.Bytes())
+	return raw.Bytes(), "sha256:" + hex.EncodeToString(sum[:])
+}

internal/declaration/declaration.go

@@ -569,6 +569,18 @@ type Process struct {
 	// that runs once does not run on a schedule, and something that is not running cannot be
 	// restarted when a file changes.
 	Schedule string `json:"schedule,omitempty"`
+
+	// Replaces names resources this declaration no longer declares that this process takes the
+	// place of (novox/hq issue 213). Such a resource is not removed with the other orphans, before
+	// anything is applied: it is removed only once this process is applied and still running a
+	// moment later, and kept when it is not. So the thing being replaced answers until the thing
+	// replacing it does — the controller moving from its container to a process is the case: the
+	// container removed first left nothing answering the mesh's verbs for as long as fetching,
+	// unpacking and starting the process took, and for ever if the process did not start.
+	//
+	// For a process that stays up; a step or a scheduled run is not running a moment later by
+	// design, so there is nothing to hand over to.
+	Replaces []string `json:"replaces,omitempty"`
 }
 
 func (d *Process) Identity() string { return d.ID }
@@ -654,6 +666,20 @@ func (d *Process) validate(where string, _ bool) []string {
 			problems = append(problems, where+": "+err.Error())
 		}
 	}
+	if len(d.Replaces) > 0 && (d.RunOnce || d.Schedule != "") {
+		problems = append(problems, where+
+			": only a process that stays up replaces something — a step or a scheduled run is not "+
+			"running a moment later, so what it replaced would be removed with nothing in its place, "+
+			"or never")
+	}
+	for _, id := range d.Replaces {
+		switch {
+		case strings.TrimSpace(id) == "":
+			problems = append(problems, where+": replaces names an empty id")
+		case id == d.ID:
+			problems = append(problems, where+": a process cannot replace itself")
+		}
+	}
 	return problems
 }
 
@@ -870,6 +896,12 @@ type Package struct {
 	ID      string `json:"id"`
 	Type    Type   `json:"type"`
 	Package string `json:"package"`
+	// Absent declares that the package is NOT installed (novox/hq ADR 0180): the host removes it
+	// when it is, and leaves a machine that never had it alone. For the one case a module replaces
+	// software the machine was found with and the operator has decided it does not come back — the
+	// firewall front end a converged machine's filter module retired. Nothing to undo when the
+	// declaration drops it: the host does not install what a declaration stopped saying is absent.
+	Absent bool `json:"absent,omitempty"`
 }
 
 func (p *Package) Identity() string { return p.ID }
@@ -946,6 +978,14 @@ type Container struct {
 	// container; a privileged container stays undeclarable.
 	Capabilities []string `json:"capabilities,omitempty"`
 
+	// Logging names where the runtime sends this container's output: "journald" sends it to the
+	// machine's journal, under the container's name, where what reads the machine's logs — its
+	// intrusion prevention first of all (novox/hq ADR 0179) — can read it the way it reads the
+	// machine's own services. Empty keeps the runtime's default, which is a file of the runtime's
+	// own that nothing but the runtime reads. Part of the spec: a container that logs elsewhere
+	// is a different container, and the runtime cannot change a running one's driver.
+	Logging string `json:"logging,omitempty"`
+
 	// Networks are networks this container also joins once created, by name — a found network a
 	// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
 	// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
@@ -997,6 +1037,24 @@ type Container struct {
 	// rather than stacked. It is exclusive with RunOnce and with restart-on: a container runs once
 	// and gates, runs on a cadence, or stays up — never two of these.
 	Schedule string `json:"schedule,omitempty"`
+
+	// WhileStopped names resources of the same module — containers — that must be held still for
+	// the duration of this step's run (novox/hq ADR 0189). The host stops each before the run and
+	// starts each again after it, **whatever the step did**: a step that failed must leave the
+	// service running, because the one real risk of this field is a window that never closes.
+	//
+	// **For the work a service cannot have done underneath it.** The artifact store's collector
+	// walks the storage and requires every writer stopped; a run-once step runs beside containers
+	// and a scheduled one is the same container again, so until this there was no way for a module
+	// to say it. The predecessor said it with a shell script, which is how the mesh inherited a
+	// store that has never collected anything.
+	//
+	// **Its own module's containers, and only on a schedule.** A module that could quiesce a
+	// neighbour could stop the mesh. And at apply time the host already has a window — the
+	// declaration is applied in order and a run-once step gates what follows — so a one-time
+	// offline job says *before*, not *instead of*; a recurring window is the case order cannot
+	// express, and the only one this serves.
+	WhileStopped []string `json:"while-stopped,omitempty"`
 }
 
 func (c *Container) Identity() string { return c.ID }
@@ -1028,6 +1086,20 @@ func (c *Container) validate(where string, _ bool) []string {
 			problems = append(problems, where+": "+err.Error())
 		}
 	}
+	// A maintenance window belongs to a recurring step (novox/hq ADR 0189). Refused on anything
+	// else here, where the field is; that it names containers of the same module, and not itself,
+	// is judged against the whole declaration (see whileStoppedNames).
+	if len(c.WhileStopped) > 0 && c.Schedule == "" {
+		problems = append(problems, where+": while-stopped needs a schedule; at apply the host "+
+			"already has a window — the declaration is applied in order and a run-once step gates "+
+			"what follows — so a one-time offline job is declared before what it works on")
+	}
+	for _, id := range c.WhileStopped {
+		if id == c.ID {
+			problems = append(problems, where+": while-stopped names "+strconv.Quote(id)+
+				", which is this step itself")
+		}
+	}
 	// The runtime's flags take addresses, and a name here would be handed to it verbatim and
 	// refused at create — after the old container was already removed. Refused on arrival instead.
 	for _, d := range c.Dns {
@@ -1051,6 +1123,10 @@ func (c *Container) validate(where string, _ bool) []string {
 				"capability's name (CAP_NET_ADMIN or NET_ADMIN)")
 		}
 	}
+	if c.Logging != "" && c.Logging != "journald" {
+		problems = append(problems, where+": logging is "+strconv.Quote(c.Logging)+", and the only place a "+
+			"container's output can be sent besides the runtime's own file is \"journald\"")
+	}
 	for _, n := range c.Networks {
 		problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
 		if n == c.Network {
@@ -1438,7 +1514,9 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 		problems = append(problems, resource.validate(where, allowActions)...)
 		d.Resources = append(d.Resources, resource)
 	}
+	problems = append(problems, checkWhileStopped(d.Resources)...)
 	problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
+	problems = append(problems, checkReplaces(d.Resources)...)
 	if env.Adoption == nil {
 		for _, r := range d.Resources {
 			if r.Kind() == TypeOpening {
@@ -1466,6 +1544,41 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 	return d, nil
 }
 
+// checkWhileStopped judges a maintenance window against the whole declaration (novox/hq ADR 0189).
+//
+// A step may hold still only a container that is **here** — in this same declaration, which is to
+// say on this machine and placed by the mesh. That is what makes it the module's own: a node's
+// declaration carries one module's resources beside another's, so the id must also be a container
+// and not a file or a directory, which there would be nothing to stop.
+//
+// Refused on arrival rather than discovered at the first fire. A window that names something the
+// host cannot stop is a window that opens at 03:00 and reports nothing until somebody reads a log.
+func checkWhileStopped(resources []Resource) []string {
+	containers := map[string]bool{}
+	for _, r := range resources {
+		if r.Kind() == TypeContainer {
+			containers[r.Identity()] = true
+		}
+	}
+	var problems []string
+	for _, r := range resources {
+		c, ok := r.(*Container)
+		if !ok {
+			continue
+		}
+		for _, id := range c.WhileStopped {
+			if containers[id] {
+				continue
+			}
+			problems = append(problems, fmt.Sprintf(
+				"resource %q: while-stopped names %q, and this declaration has no container by "+
+					"that id. A step may hold still only a container placed on this machine "+
+					"beside it", c.ID, id))
+		}
+	}
+	return problems
+}
+
 func strictDecode(raw []byte, into any) error {
 	// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
 	// field the host does not know is a thing the control plane believes it asked for.
@@ -1622,3 +1735,36 @@ func stripComments(raw []byte) []byte {
 	}
 	return []byte(strings.Join(kept, "\n"))
 }
+
+// checkReplaces refuses a `replaces` naming something this same declaration still declares, or
+// named by two processes (novox/hq issue 213). What is replaced is what the declaration no longer
+// says — a resource still declared is applied, not handed over, and one handed to two replacements
+// would go when the first of them came up, whatever became of the second.
+func checkReplaces(resources []Resource) []string {
+	declared := map[string]bool{}
+	for _, r := range resources {
+		declared[r.Identity()] = true
+	}
+	var problems []string
+	by := map[string]string{}
+	for _, r := range resources {
+		p, ok := r.(*Process)
+		if !ok {
+			continue
+		}
+		for _, id := range p.Replaces {
+			if declared[id] {
+				problems = append(problems, fmt.Sprintf(
+					"resource %q: replaces %q, which this declaration still declares — a process "+
+						"replaces what the declaration no longer says", p.ID, id))
+			}
+			if other, taken := by[id]; taken && other != p.ID {
+				problems = append(problems, fmt.Sprintf(
+					"resource %q: replaces %q, which %q replaces too — one thing has one replacement",
+					p.ID, id, other))
+			}
+			by[id] = p.ID
+		}
+	}
+	return problems
+}

internal/declaration/declaration_test.go

@@ -524,3 +524,24 @@ func TestACapabilityIsNamedOrRefused(t *testing.T) {
 		}
 	}
 }
+
+// A container may send its output to the machine's journal, and nowhere else but the runtime's own
+// file (novox/hq ADR 0179): what reads the machine's logs then reads the container's too.
+func TestAContainerMayLogToTheJournalAndNowhereElse(t *testing.T) {
+	image := "postgres@sha256:" + strings.Repeat("a", 64)
+	d, err := Parse([]byte(`{"declaration":1,"resources":[
+	  {"id":"front","type":"container","name":"front","image":"` + image + `","logging":"journald"}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := d.Resources[0].(*Container).Logging; got != "journald" {
+		t.Fatalf("logging read as %q", got)
+	}
+	for _, bad := range []string{`"syslog"`, `"none"`, `"json-file"`} {
+		if _, err := Parse([]byte(`{"declaration":1,"resources":[
+		  {"id":"front","type":"container","name":"front","image":"` + image + `","logging":` + bad + `}]}`)); err == nil {
+			t.Errorf("%s was accepted as a place to log", bad)
+		}
+	}
+}

internal/declaration/replaces_test.go

@@ -0,0 +1,63 @@
+package declaration
+
+import (
+	"strings"
+	"testing"
+)
+
+// novox/hq issue 213: a process says what it takes the place of, so the host can keep the old one
+// answering until the new one does. What it may name is narrow, and each refusal is said here.
+
+const aReplacingProcess = `{"id":"m.controller","type":"process","name":"m","source":"https://store.invalid/m",
+  "digest":"sha256:` + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + `","run":["./m","serve"]%s}`
+
+func replacing(extra string) string {
+	return `{"declaration":1,"resources":[` + strings.Replace(aReplacingProcess, "%s", extra, 1) + `]}`
+}
+
+func TestAProcessMayNameWhatItReplaces(t *testing.T) {
+	d, err := Parse([]byte(replacing(`,"replaces":["m.server"]`)))
+	if err != nil {
+		t.Fatalf("a process replacing an undeclared container was refused: %v", err)
+	}
+	if p := d.Resources[0].(*Process); len(p.Replaces) != 1 || p.Replaces[0] != "m.server" {
+		t.Fatalf("replaces was not read: %+v", p)
+	}
+}
+
+// What is replaced is what the declaration no longer says. A resource still declared is applied,
+// and handing it over as well would remove something the same declaration asks to keep.
+func TestAProcessMayNotReplaceSomethingStillDeclared(t *testing.T) {
+	raw := `{"declaration":1,"resources":[
+	  {"id":"m.server","type":"directory","path":"/tmp/x"},
+	  ` + strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + `]}`
+	if _, err := Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "still declares") {
+		t.Fatalf("a process replacing a declared resource was accepted: %v", err)
+	}
+}
+
+func TestOnlyAProcessThatStaysUpReplacesAnything(t *testing.T) {
+	for _, mode := range []string{`,"run-once":true`, `,"schedule":"0 3 * * *"`} {
+		if _, err := Parse([]byte(replacing(mode + `,"replaces":["m.server"]`))); err == nil {
+			t.Errorf("a process with %s was allowed to replace something", mode)
+		}
+	}
+}
+
+func TestAProcessCannotReplaceItselfOrNothing(t *testing.T) {
+	for _, bad := range []string{`,"replaces":["m.controller"]`, `,"replaces":[""]`} {
+		if _, err := Parse([]byte(replacing(bad))); err == nil {
+			t.Errorf("replaces %s was accepted", bad)
+		}
+	}
+}
+
+func TestOneThingHasOneReplacement(t *testing.T) {
+	second := strings.NewReplacer(`"id":"m.controller"`, `"id":"m.other"`, `"name":"m"`, `"name":"other"`).
+		Replace(strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1))
+	raw := `{"declaration":1,"resources":[` +
+		strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + "," + second + `]}`
+	if _, err := Parse([]byte(raw)); err == nil {
+		t.Fatal("two processes replacing one resource were accepted")
+	}
+}

internal/firewall/filters.go

@@ -179,27 +179,18 @@ func legacyFilters(rules, tool string, ufwActive bool) []Filter {
 			}
 		}
 	}
-	var entered func(chain string, seen map[string]bool) bool
+	// A chain of refusals is a ban list when every refusal names the sources it refuses and the
-	entered = func(chain string, seen map[string]bool) bool {
+	// chain accepts nothing — the same rule the nftables side applies, and no more.
-		if seen[chain] || accepting[chain] || len(jumpedFrom[chain]) == 0 {
+	//
-			return false
+	// **The policy of the chains that jump to it says nothing about what it is.** An earlier cut
-		}
+	// required every path into the chain to come from a built-in whose policy accepts, and the
-		seen[chain] = true
+	// mesh's own intrusion prevention then read as a foreign rule set on the home server: its ban
-		for _, from := range jumpedFrom[chain] {
+	// chain hangs off the container runtime's user chain as well as INPUT, and that machine's
-			if p, builtIn := policy[from]; builtIn {
+	// forward policy is DROP because the runtime set it. The machine reported "NOT the mesh alone"
-				if p != "ACCEPT" {
+	// about a chain the mesh had just written (novox/hq ADR 0186). The policy is already classified
-					return false
+	// where it belongs — as the runtime's — so requiring it here counted it twice.
-				}
-				continue
-			}
-			if !entered(from, seen) {
-				return false
-			}
-		}
-		return true
-	}
 	ban := func(chain, line string) bool {
-		return bansSources(line) && entered(chain, map[string]bool{})
+		return bansSources(line) && !accepting[chain] && len(jumpedFrom[chain]) > 0
 	}
 	type seen struct {
 		owner string
@@ -319,8 +310,17 @@ func Active(ctx context.Context, run Runner) bool {
 	return err == nil && statusActive(out)
 }
 
+// Installed says whether ufw is on this machine at all: a command that is not there is a front end
+// that was uninstalled (novox/hq ADR 0180), not one that is silent.
+func Installed(ctx context.Context, run Runner) bool {
+	_, err := run(ctx, "ufw", "status")
+	return !missing(err)
+}
+
 // Retirements of a found firewall, as the host records them.
 const (
 	RetiredByMesh  = "mesh"
 	RetiredFoundSo = "found-inactive"
+	// RetiredRemoved is a front end uninstalled by the module that replaced it (ADR 0180).
+	RetiredRemoved = "removed"
 )

internal/firewall/neighbour_ban_test.go

@@ -0,0 +1,60 @@
+package firewall
+
+import (
+	"os"
+	"testing"
+)
+
+// The mesh's own ban list is a ban wherever it hangs (novox/hq ADR 0186).
+//
+// Captured from the home server after the intrusion prevention had banned four addresses: its ban
+// chain is jumped to from INPUT, whose policy accepts, and from the container runtime's user chain,
+// which hangs off a FORWARD the runtime set to DROP. Requiring every path to come from an accepting
+// built-in made the machine report "NOT the mesh alone" about a chain the mesh had just written.
+func TestTheMeshsOwnBanChainIsABanBehindADroppingForward(t *testing.T) {
+	legacy, err := os.ReadFile("testdata/home-server-bans-S.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	filters := Filters("", map[string]string{"iptables-legacy": string(legacy)}, false)
+
+	var ban, other []string
+	for _, f := range filters {
+		switch f.Owner {
+		case OwnerBan:
+			ban = append(ban, f.Where)
+		case OwnerOther:
+			other = append(other, f.Where)
+		}
+	}
+	if len(other) > 0 {
+		t.Errorf("the machine reports %v as rule sets the mesh did not write", other)
+	}
+	found := false
+	for _, w := range ban {
+		if w == "chain f2b-route-proxy (iptables-legacy)" {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("the intrusion prevention's own chain was not read as a ban; bans were %v", ban)
+	}
+	if !Alone(filters) {
+		t.Error("a machine filtered by the mesh and its own bans does not read as the mesh alone")
+	}
+}
+
+// A chain that accepts anything is doing more than banning, and is still not a ban — which is what
+// keeps a predecessor's allow-and-drop chain classified as something the operator must look at.
+func TestAChainThatAcceptsIsNotABan(t *testing.T) {
+	rules := "-P INPUT ACCEPT\n" +
+		"-A INPUT -j HAL-MESH-ONLY\n" +
+		"-N HAL-MESH-ONLY\n" +
+		"-A HAL-MESH-ONLY -s 10.0.0.0/8 -j ACCEPT\n" +
+		"-A HAL-MESH-ONLY -s 203.0.113.7/32 -j DROP\n"
+	for _, f := range Filters("", map[string]string{"iptables-legacy": rules}, false) {
+		if f.Where == "chain HAL-MESH-ONLY (iptables-legacy)" && f.Owner != OwnerOther {
+			t.Errorf("a chain that accepts was classified as %s", f.Owner)
+		}
+	}
+}

internal/firewall/testdata/home-server-bans-S.txt

@@ -0,0 +1,147 @@
+-P INPUT ACCEPT
+-P FORWARD DROP
+-P OUTPUT ACCEPT
+-N DOCKER
+-N DOCKER-BRIDGE
+-N DOCKER-CT
+-N DOCKER-FORWARD
+-N DOCKER-INTERNAL
+-N DOCKER-USER
+-N f2b-route-proxy
+-N ufw-after-forward
+-N ufw-after-input
+-N ufw-after-logging-forward
+-N ufw-after-logging-input
+-N ufw-after-logging-output
+-N ufw-after-output
+-N ufw-before-forward
+-N ufw-before-input
+-N ufw-before-logging-forward
+-N ufw-before-logging-input
+-N ufw-before-logging-output
+-N ufw-before-output
+-N ufw-reject-forward
+-N ufw-reject-input
+-N ufw-reject-output
+-N ufw-track-forward
+-N ufw-track-input
+-N ufw-track-output
+-A INPUT -p tcp -j f2b-route-proxy
+-A INPUT -j ufw-before-logging-input
+-A INPUT -j ufw-before-input
+-A INPUT -j ufw-after-input
+-A INPUT -j ufw-after-logging-input
+-A INPUT -j ufw-reject-input
+-A INPUT -j ufw-track-input
+-A FORWARD -j DOCKER-USER
+-A FORWARD -j DOCKER-FORWARD
+-A FORWARD -j ufw-before-logging-forward
+-A FORWARD -j ufw-before-forward
+-A FORWARD -j ufw-after-forward
+-A FORWARD -j ufw-after-logging-forward
+-A FORWARD -j ufw-reject-forward
+-A FORWARD -j ufw-track-forward
+-A OUTPUT -j ufw-before-logging-output
+-A OUTPUT -j ufw-before-output
+-A OUTPUT -j ufw-after-output
+-A OUTPUT -j ufw-after-logging-output
+-A OUTPUT -j ufw-reject-output
+-A OUTPUT -j ufw-track-output
+-A DOCKER -d 172.17.0.18/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8686 -j ACCEPT
+-A DOCKER -d 172.17.0.14/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8989 -j ACCEPT
+-A DOCKER -d 172.17.0.15/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 7878 -j ACCEPT
+-A DOCKER -d 172.17.0.5/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9117 -j ACCEPT
+-A DOCKER -d 172.17.0.13/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6789 -j ACCEPT
+-A DOCKER -d 172.19.0.2/32 ! -i br-32062158f584 -o br-32062158f584 -p tcp -m tcp --dport 8080 -j ACCEPT
+-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 5432 -j ACCEPT
+-A DOCKER -d 172.27.0.2/32 ! -i br-0910a98c6158 -o br-0910a98c6158 -p tcp -m tcp --dport 5678 -j ACCEPT
+-A DOCKER -d 172.17.0.21/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3579 -j ACCEPT
+-A DOCKER -d 172.17.0.19/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8181 -j ACCEPT
+-A DOCKER -d 172.17.0.17/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8787 -j ACCEPT
+-A DOCKER -d 172.17.0.16/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6767 -j ACCEPT
+-A DOCKER -d 172.17.0.12/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
+-A DOCKER -d 172.17.0.11/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT
+-A DOCKER -d 172.17.0.10/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9443 -j ACCEPT
+-A DOCKER -d 172.17.0.10/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9000 -j ACCEPT
+-A DOCKER -d 172.17.0.9/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
+-A DOCKER -d 172.17.0.7/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 1880 -j ACCEPT
+-A DOCKER -d 172.28.0.2/32 ! -i br-b11461b5b028 -o br-b11461b5b028 -p tcp -m tcp --dport 80 -j ACCEPT
+-A DOCKER -d 172.23.0.14/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 6543 -j ACCEPT
+-A DOCKER -d 172.23.0.14/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 5432 -j ACCEPT
+-A DOCKER -d 172.23.0.5/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 8000 -j ACCEPT
+-A DOCKER -d 172.26.0.3/32 ! -i br-b0fec361ccaa -o br-b0fec361ccaa -p tcp -m tcp --dport 6167 -j ACCEPT
+-A DOCKER -d 172.26.0.2/32 ! -i br-b0fec361ccaa -o br-b0fec361ccaa -p tcp -m tcp --dport 80 -j ACCEPT
+-A DOCKER -d 172.17.0.8/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8000 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10001 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8880 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8843 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8443 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8080 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6789 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 5514 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 3478 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 1900 -j ACCEPT
+-A DOCKER -d 172.25.0.3/32 ! -i br-b98821f7dc38 -o br-b98821f7dc38 -p tcp -m tcp --dport 8000 -j ACCEPT
+-A DOCKER -d 172.18.0.3/32 ! -i br-442a0bfc65f8 -o br-442a0bfc65f8 -p tcp -m tcp --dport 1433 -j ACCEPT
+-A DOCKER -d 172.20.0.3/32 ! -i br-afa37ac8b33d -o br-afa37ac8b33d -p tcp -m tcp --dport 8081 -j ACCEPT
+-A DOCKER -d 172.20.0.3/32 ! -i br-afa37ac8b33d -o br-afa37ac8b33d -p tcp -m tcp --dport 1883 -j ACCEPT
+-A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8086 -j ACCEPT
+-A DOCKER -d 172.21.0.2/32 ! -i br-df15d8e19ec7 -o br-df15d8e19ec7 -p tcp -m tcp --dport 6379 -j ACCEPT
+-A DOCKER -d 172.30.0.3/32 ! -i br-521eab9a3a5e -o br-521eab9a3a5e -p tcp -m tcp --dport 8283 -j ACCEPT
+-A DOCKER -d 172.17.0.3/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
+-A DOCKER ! -i br-32062158f584 -o br-32062158f584 -j DROP
+-A DOCKER ! -i docker0 -o docker0 -j DROP
+-A DOCKER ! -i br-521eab9a3a5e -o br-521eab9a3a5e -j DROP
+-A DOCKER ! -i br-df15d8e19ec7 -o br-df15d8e19ec7 -j DROP
+-A DOCKER ! -i br-afa37ac8b33d -o br-afa37ac8b33d -j DROP
+-A DOCKER ! -i br-442a0bfc65f8 -o br-442a0bfc65f8 -j DROP
+-A DOCKER ! -i br-b98821f7dc38 -o br-b98821f7dc38 -j DROP
+-A DOCKER ! -i br-b0fec361ccaa -o br-b0fec361ccaa -j DROP
+-A DOCKER ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -j DROP
+-A DOCKER ! -i br-b11461b5b028 -o br-b11461b5b028 -j DROP
+-A DOCKER ! -i br-2df4e541b877 -o br-2df4e541b877 -j DROP
+-A DOCKER ! -i br-0910a98c6158 -o br-0910a98c6158 -j DROP
+-A DOCKER-BRIDGE -o br-32062158f584 -j DOCKER
+-A DOCKER-BRIDGE -o docker0 -j DOCKER
+-A DOCKER-BRIDGE -o br-521eab9a3a5e -j DOCKER
+-A DOCKER-BRIDGE -o br-df15d8e19ec7 -j DOCKER
+-A DOCKER-BRIDGE -o br-afa37ac8b33d -j DOCKER
+-A DOCKER-BRIDGE -o br-442a0bfc65f8 -j DOCKER
+-A DOCKER-BRIDGE -o br-b98821f7dc38 -j DOCKER
+-A DOCKER-BRIDGE -o br-b0fec361ccaa -j DOCKER
+-A DOCKER-BRIDGE -o br-66ffa5c1cba5 -j DOCKER
+-A DOCKER-BRIDGE -o br-b11461b5b028 -j DOCKER
+-A DOCKER-BRIDGE -o br-2df4e541b877 -j DOCKER
+-A DOCKER-BRIDGE -o br-0910a98c6158 -j DOCKER
+-A DOCKER-CT -o br-32062158f584 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-521eab9a3a5e -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-df15d8e19ec7 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-afa37ac8b33d -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-442a0bfc65f8 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-b98821f7dc38 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-b0fec361ccaa -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-66ffa5c1cba5 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-b11461b5b028 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-2df4e541b877 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-0910a98c6158 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-FORWARD -j DOCKER-CT
+-A DOCKER-FORWARD -j DOCKER-INTERNAL
+-A DOCKER-FORWARD -j DOCKER-BRIDGE
+-A DOCKER-FORWARD -i br-32062158f584 -j ACCEPT
+-A DOCKER-FORWARD -i docker0 -j ACCEPT
+-A DOCKER-FORWARD -i br-521eab9a3a5e -j ACCEPT
+-A DOCKER-FORWARD -i br-df15d8e19ec7 -j ACCEPT
+-A DOCKER-FORWARD -i br-afa37ac8b33d -j ACCEPT
+-A DOCKER-FORWARD -i br-442a0bfc65f8 -j ACCEPT
+-A DOCKER-FORWARD -i br-b98821f7dc38 -j ACCEPT
+-A DOCKER-FORWARD -i br-b0fec361ccaa -j ACCEPT
+-A DOCKER-FORWARD -i br-66ffa5c1cba5 -j ACCEPT
+-A DOCKER-FORWARD -i br-b11461b5b028 -j ACCEPT
+-A DOCKER-FORWARD -i br-2df4e541b877 -j ACCEPT
+-A DOCKER-FORWARD -i br-0910a98c6158 -j ACCEPT
+-A DOCKER-USER -p tcp -j f2b-route-proxy
+-A f2b-route-proxy -s 13.70.107.184/32 -j REJECT --reject-with icmp-port-unreachable
+-A f2b-route-proxy -s 45.138.12.51/32 -j REJECT --reject-with icmp-port-unreachable
+-A f2b-route-proxy -s 20.214.191.94/32 -j REJECT --reject-with icmp-port-unreachable
+-A f2b-route-proxy -j RETURN

internal/system/alpine.go

@@ -46,6 +46,11 @@ func (a alpine) PackageInstalled(ctx context.Context, run Runner, name string) (
 	return strings.TrimSpace(out) != "", nil
 }
 
+func (alpine) RemovePackage(ctx context.Context, run Runner, name string) error {
+	_, err := run(ctx, "apk", "del", name)
+	return err
+}
+
 func (alpine) InstallPackage(ctx context.Context, run Runner, name string) error {
 	_, err := run(ctx, "apk", "add", "--no-cache", name)
 	return err

internal/system/android.go

@@ -65,6 +65,10 @@ func (a android) InstallPackage(context.Context, Runner, string) error {
 	return fmt.Errorf("%w: package", ErrUnsupported)
 }
 
+func (a android) RemovePackage(context.Context, Runner, string) error {
+	return fmt.Errorf("%w: package", ErrUnsupported)
+}
+
 func (a android) ServiceState(context.Context, Runner, string) (string, error) {
 	return "", fmt.Errorf("%w: service (init is not reachable without root)", ErrUnsupported)
 }

internal/system/arch.go

@@ -3,7 +3,10 @@ package system
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 )
@@ -39,6 +42,14 @@ func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bo
 	return true, nil
 }
 
+// RemovePackage removes one package and nothing it depends on: `-R`, not `-Rs`, because what else
+// relied on a dependency is not this declaration's to know. pacman keeps a configuration file the
+// operator changed as `.pacsave`, which is what "never flushed" comes to once the front end is gone.
+func (arch) RemovePackage(ctx context.Context, run Runner, name string) error {
+	_, err := run(ctx, "pacman", "-R", "--noconfirm", name)
+	return err
+}
+
 func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name)
 	if err == nil {
@@ -48,42 +59,119 @@ func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	// **The package manager's own words, and a name for the case that looks like a bug in the
 	// declaration and is not.** A stale index asks the mirrors for a version they have already
 	// superseded and gets a 404 from every one of them — so the package exists, the declaration is
-	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002).
+	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
+	// old as the index fails one step later, on the signature of whatever a mirror still had.
+	//
+	// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
+	// the error rather than the output; this classifier read the output alone and so never saw a
+	// single "failed retrieving file", and the control node reported a ten-week-old database as
+	// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
 	//
 	// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
 	// libraries this machine does not have: a partial upgrade, which Arch does not support and
 	// which breaks the machine in a way that surfaces much later as something unrelated. The
 	// remedy is a full upgrade, and it is a decision about the whole machine rather than
-	// something to do silently in the middle of applying one resource.
+	// something to do silently in the middle of applying one resource. Whose decision, and on
-	//
+	// what schedule, is issue 205's question; until it is answered the host says what it sees.
-	// So this says which of the two it is looking at. A declaration that is wrong and a machine
+	said := strings.TrimSpace(out + "\n" + err.Error())
-	// that is out of date fail identically otherwise, and they are fixed in completely different
+	switch classifyInstallFailure(said) {
-	// places.
+	case installStale:
-	if staleIndex(out) {
 		return fmt.Errorf(
-			"%s could not be fetched from any mirror, which is what a stale package index looks "+
+			"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
-				"like: this machine is asking for a version the mirrors have replaced. The "+
+				"the package database on this machine is %s and the mirrors no longer serve what it "+
-				"package and the declaration are probably both fine. It is fixed by upgrading "+
+				"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
-				"the machine, not by this host syncing one package — that would be a partial "+
+				"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
-				"upgrade, which this distribution does not support.\n\n%s",
+				"host does not sync one package by itself, because on this distribution that is a "+
-			name, strings.TrimSpace(out))
+				"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
+			name, syncDatabaseAge(), name, said)
+	case installMirrors:
+		return fmt.Errorf(
+			"no mirror could be reached to fetch %s, and the package database on this machine is "+
+				"%s: this reads as the mirrors or the network, not as this machine being out of "+
+				"date — try again when they answer.\n\n%s",
+			name, syncDatabaseAge(), said)
 	}
 	return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
 }
 
-// staleIndex reports whether a failed install looks like the machine's view being old rather than
+// How a failed install is read, from what the package manager said.
-// the package being wrong.
+type installFailure int
-//
+
-// By what the package manager said, because there is nothing else to go on: the exit code is the
+const (
-// same for both.
+	installOther installFailure = iota
-func staleIndex(out string) bool {
+	// installStale: the machine's package database or keyring is older than what the mirrors
-	said := strings.ToLower(out)
+	// serve — every mirror 404s the file the database names, or a package that did arrive fails
-	if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") {
+	// its signature against a keyring that never saw the key.
-		return false
+	installStale
+	// installMirrors: no mirror could be reached at all, and nothing says the database is old.
+	installMirrors
+)
+
+// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
+// code is the same for every one of these.
+func classifyInstallFailure(said string) installFailure {
+	lower := strings.ToLower(said)
+	gone := strings.Count(lower, "returned error: 404")
+	fetching := strings.Contains(lower, "failed retrieving file")
+	badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
+		strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
+		strings.Contains(lower, "is unknown trust") ||
+		strings.Contains(lower, "could not be looked up remotely")
+	switch {
+	case badSignature:
+		return installStale
+	case fetching && gone > 0:
+		// Every mirror, not one: a single mirror failing is an ordinary transient thing and
+		// retrying is the answer. pacman walks its whole mirror list before giving up, so more
+		// than one 404 among the lines is the index being old rather than one host being wrong.
+		if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
+			!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
+			return installStale
 		}
-	// Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying
+		return installMirrors
-	// is the answer; every one of them saying the file is gone is the index being old.
+	case fetching:
-	return strings.Contains(said, "error") || strings.Count(said, "404") > 1
+		return installMirrors
+	}
+	return installOther
+}
+
+// staleIndex is the yes-or-no form older callers and tests use.
+func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
+
+// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
+// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
+// install so a ten-week-old database is told apart from a mirror outage by reading one line.
+//
+// A variable so a test can say what the machine's database looks like without having one.
+var syncDatabaseAge = func() string {
+	entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
+	if err != nil || len(entries) == 0 {
+		return "of unknown age (no sync database found under /var/lib/pacman/sync)"
+	}
+	var newest time.Time
+	for _, e := range entries {
+		info, err := os.Stat(e)
+		if err == nil && info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	if newest.IsZero() {
+		return "of unknown age"
+	}
+	return describeAge(newest, time.Now())
+}
+
+// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
+func describeAge(when, now time.Time) string {
+	days := int(now.Sub(when).Hours() / 24)
+	span := fmt.Sprintf("%d days old", days)
+	switch {
+	case days < 1:
+		span = "less than a day old"
+	case days >= 14:
+		span = fmt.Sprintf("%d weeks old", days/7)
+	}
+	return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
 }
 
 // ServiceState reads what systemd says about a unit.

internal/system/arch_install_test.go

@@ -0,0 +1,115 @@
+package system
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+// pacman's own words from the control node on 2026-10-02 (novox/hq 04-ISSUES/205): every mirror
+// 404s the versioned file a ten-week-old database names, and the one copy that arrives fails its
+// signature. Errors are pacman's stderr, which the runner folds into the error, not the output.
+const staleStderr = `error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.hetzner.com : The requested URL returned error: 404
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.rackspace.com : The requested URL returned error: 404
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from arch.lucassymons.net : Could not resolve host: arch.lucassymons.net
+warning: fatal error from arch.lucassymons.net, skipping for the remainder of this transaction
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirrors.cqu.edu.cn : Connection timed out after 10002 milliseconds
+error: nodejs: signature from "Bert Peters (packager key) <bertptrs@archlinux.org>" is invalid
+error: failed to commit transaction (invalid or corrupted package (PGP signature))`
+
+const staleStdout = `resolving dependencies...
+looking for conflicting packages...
+
+Packages (4) ada-3.4.4-1  c-ares-1.34.8-1  simdjson-1:4.6.4-1  nodejs-26.5.0-1
+
+:: Retrieving packages...
+ nodejs-26.5.0-1-x86_64 downloading...
+checking keyring...
+checking package integrity...
+:: File /var/cache/pacman/pkg/nodejs-26.5.0-1-x86_64.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)).
+Errors occurred, no packages were upgraded.`
+
+// A runner that behaves as ExecRunner does on failure: stdout as the output, stderr in the error.
+func pacmanFailing(stdout, stderr string) Runner {
+	return func(_ context.Context, name string, args ...string) (string, error) {
+		return stdout, errors.New(name + " exited 1: " + stderr)
+	}
+}
+
+func TestAStaleDatabaseIsSaidAsOneWithItsAgeAndTheRemedy(t *testing.T) {
+	was := syncDatabaseAge
+	defer func() { syncDatabaseAge = was }()
+	syncDatabaseAge = func() string {
+		return describeAge(time.Date(2026, 7, 24, 16, 56, 0, 0, time.UTC), time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC))
+	}
+
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing(staleStdout, staleStderr), "nodejs")
+	if err == nil {
+		t.Fatal("a failed install must fail")
+	}
+	for _, want := range []string{
+		"the package database on this machine is from 2026-07-24, 10 weeks old",
+		"stale package index",
+		"upgrading the machine — a full upgrade (`pacman -Syu`) by its operator — before the mesh can install nodejs",
+		"partial upgrade",
+		"returned error: 404", // pacman's own words follow
+	} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the error does not say %q:\n%s", want, err)
+		}
+	}
+	if strings.Contains(err.Error(), "mirrors or the network") {
+		t.Errorf("a stale database must not be read as a mirror outage:\n%s", err)
+	}
+}
+
+// The same 404s read from stderr alone — the half this classifier used to be blind to.
+func TestTheClassifierReadsWhatPacmanWroteToStderr(t *testing.T) {
+	if classifyInstallFailure(staleStderr) != installStale {
+		t.Fatal("every mirror 404ing the named file is a stale database")
+	}
+	if classifyInstallFailure(staleStdout) != installStale {
+		t.Fatal("a corrupted-signature line alone is a stale keyring")
+	}
+	if classifyInstallFailure("") != installOther {
+		t.Fatal("nothing said is nothing classified")
+	}
+}
+
+func TestAnUnreachableMirrorWithAFreshDatabaseIsAMirrorProblem(t *testing.T) {
+	was := syncDatabaseAge
+	defer func() { syncDatabaseAge = was }()
+	syncDatabaseAge = func() string { return "from 2026-10-02, less than a day old" }
+	outage := `error: failed retrieving file 'core.db' from mirror.hetzner.com : Could not resolve host: mirror.hetzner.com
+error: failed retrieving file 'core.db' from mirror.rackspace.com : Connection timed out after 10001 milliseconds
+error: failed to synchronize all databases (failed to retrieve some files)`
+	if classifyInstallFailure(outage) != installMirrors {
+		t.Fatal("no mirror answering, no 404, no signature fault: the mirrors, not the machine")
+	}
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", outage), "nodejs")
+	if err == nil || !strings.Contains(err.Error(), "mirrors or the network") || !strings.Contains(err.Error(), "less than a day old") {
+		t.Errorf("a mirror outage is said as one, with the database's age beside it:\n%v", err)
+	}
+}
+
+func TestAFailureThatIsNeitherKeepsPacmansWords(t *testing.T) {
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", "error: target not found: nodejsx"), "nodejsx")
+	if err == nil || !strings.Contains(err.Error(), "target not found") || strings.Contains(err.Error(), "package database on this machine") {
+		t.Errorf("an unknown package is pacman's own error, not a stale database:\n%v", err)
+	}
+}
+
+func TestDescribeAge(t *testing.T) {
+	now := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
+	for when, want := range map[time.Time]string{
+		now.Add(-2 * time.Hour):       "less than a day old",
+		now.Add(-5 * 24 * time.Hour):  "5 days old",
+		now.Add(-71 * 24 * time.Hour): "10 weeks old",
+	} {
+		if got := describeAge(when, now); !strings.HasSuffix(got, want) {
+			t.Errorf("%s: got %q, want suffix %q", when, got, want)
+		}
+	}
+}

internal/system/lookup_exit_test.go

@@ -0,0 +1,40 @@
+package system
+
+import (
+	"context"
+	"errors"
+	"os/exec"
+	"testing"
+)
+
+// A user that does not exist yet is "absent", in the words the host's own runner uses
+// ("getent exited 2"), not only Go's ("exit status 2"). Matched as text, the runner's wording read as
+// a user database that did not answer, and the controller's account was never created.
+func TestAMissingUserIsAbsentInTheRunnersWords(t *testing.T) {
+	for _, words := range []string{"getent exited 2: ", "exit status 2"} {
+		run := func(context.Context, string, ...string) (string, error) { return "", errors.New(words) }
+		_, found, err := LookUpUser(context.Background(), run, "nobody-here")
+		if err != nil || found {
+			t.Errorf("%q: found %v, err %v; want absent", words, found, err)
+		}
+	}
+	run := func(context.Context, string, ...string) (string, error) { return "", errors.New("getent exited 1: ") }
+	if _, _, err := LookUpUser(context.Background(), run, "x"); err == nil {
+		t.Error("a database that failed read as an answer")
+	}
+}
+
+// The real command, through a real exit: getent's code for a key not found.
+func TestAMissingUserIsAbsentFromTheRealGetent(t *testing.T) {
+	if _, err := exec.LookPath("getent"); err != nil {
+		t.Skip("no getent here")
+	}
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		out, err := exec.CommandContext(ctx, name, args...).Output()
+		return string(out), err
+	}
+	_, found, err := LookUpUser(context.Background(), run, "mesh-no-such-user-0b1f")
+	if err != nil || found {
+		t.Errorf("found %v, err %v; want absent", found, err)
+	}
+}

internal/system/system.go

@@ -19,6 +19,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/novox/mesh-host/internal/declaration"
@@ -51,6 +54,9 @@ type System interface {
 
 	PackageInstalled(ctx context.Context, run Runner, name string) (bool, error)
 	InstallPackage(ctx context.Context, run Runner, name string) error
+	// RemovePackage uninstalls one package, leaving its dependencies and anything the operator
+	// changed in its configuration where the package manager leaves them (novox/hq ADR 0180).
+	RemovePackage(ctx context.Context, run Runner, name string) error
 
 	// ServiceState is "running" or "stopped". A unit that does not exist is an error, never
 	// "stopped" — reporting absence as satisfaction is the fault this host exists to prevent.
@@ -92,8 +98,10 @@ func LookUpUser(ctx context.Context, run Runner, name string) (Login, bool, erro
 	out, err := run(ctx, "getent", "passwd", name)
 	if err != nil {
 		// getent's own convention: 2 means the key was not found, which is the only failure that
-		// means "no such user".
+		// means "no such user". Asked of the exit code: matched as text, it looked for Go's wording
-		if strings.Contains(err.Error(), "exit status 2") {
+		// ("exit status 2") while the host's runner says "getent exited 2", so a user that did not
+		// exist yet read as a database that did not answer, and no account was ever created.
+		if code, ok := ExitCode(err); ok && code == 2 {
 			return Login{}, false, nil
 		}
 		return Login{}, false, fmt.Errorf(
@@ -218,3 +226,27 @@ func For(name string) (System, error) {
 func All() []System {
 	return []System{arch{}, alpine{}, android{}}
 }
+
+// ExitCode is the code a command exited with, when err says one: from the exit itself where the
+// runner kept it, else from the words either runner shape uses ("exit status N", "<cmd> exited N").
+func ExitCode(err error) (int, bool) {
+	if err == nil {
+		return 0, false
+	}
+	var exit *exec.ExitError
+	if errors.As(err, &exit) {
+		return exit.ExitCode(), true
+	}
+	if m := exitWords.FindStringSubmatch(err.Error()); len(m) == 3 {
+		for _, g := range m[1:] {
+			if g != "" {
+				if n, convErr := strconv.Atoi(g); convErr == nil {
+					return n, true
+				}
+			}
+		}
+	}
+	return 0, false
+}
+
+var exitWords = regexp.MustCompile(`exit status (\d+)|exited (\d+)`)