Hand a replaced resource over to the process that replaces it (hq issue 213)

The controller moves from a container to a process on the one machine
that runs it (novox/hq issue 213). Every orphan is removed before anything
is applied, so the container would go first and nothing would answer the
mesh's verbs while the process was fetched, unpacked and started — and
never again, if it did not start.

- a process may say what it `replaces`: resources the declaration no
  longer declares. Such an orphan is kept through the up-front sweep and
  removed right after the process applied and is up: active and running
  at two looks ten seconds apart, the same main process, no restart in
  between (stricter than ADR 0184's second look, which reads a unit
  waiting to restart as running). If the process failed, was skipped
  behind its module's step, or is not running, the orphan stays running
  and recorded, reported kept, and the next apply hands it over.
  Refused: naming something still declared, itself, an empty id, one
  thing named by two processes, and `replaces` on a step or a schedule.
- beyond #85's oneshot unit for a step: a step written ./name runs its
  own bundle's binary (tested), and is started, never enabled.
- a run-once process that fails gates its module, as a run-once
  container already did, so a version whose preparation failed is not
  started.
- an unchanged run-once process is not run again, and an unchanged
  scheduled one is kept up by its timer: both were "a daemon that had
  stopped" and were started on every apply.

examples/foundation-first-node-nats.lock

@@ -161,7 +161,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.mesh-controller.tool.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

internal/apply/apply.go

@@ -247,6 +247,20 @@ func ApplyKeeping(
 	// orphan is removed, and if a removal then fails the guard is already up. A stale opening on an
 	// adopted node is removed as any orphan is.
 	var protecting, orphans []store.Applied
+	// **What a declared process replaces is handed over, not removed first** (novox/hq issue 213).
+	// Every other orphan goes before anything is applied; one a process names under `replaces` is
+	// kept until that process is applied and still running a moment later, so whatever it was —
+	// the controller's container — answers until its replacement does, and goes on answering if
+	// the replacement never comes up.
+	replacedBy := map[string]string{}
+	for _, r := range d.Resources {
+		if p, ok := r.(*declaration.Process); ok {
+			for _, id := range p.Replaces {
+				replacedBy[id] = p.ID
+			}
+		}
+	}
+	handover := map[string][]store.Applied{}
 	for _, orphan := range known.Orphans(declared, origin) {
 		if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
 			protecting = append(protecting, orphan)
@@ -264,6 +278,10 @@ func ApplyKeeping(
 			log(fmt.Sprintf("  kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
 			continue
 		}
+		if by, replaced := replacedBy[orphan.ID]; replaced {
+			handover[by] = append(handover[by], orphan)
+			continue
+		}
 		orphans = append(orphans, orphan)
 	}
 	ordered := d.Resources
@@ -517,6 +535,11 @@ func ApplyKeeping(
 			if c, ok := resource.(*declaration.Container); ok && c.RunOnce {
 				gates = true
 			}
+			// And a run-once process, which is the same step hosted as a unit: a version whose
+			// preparation did not complete must not be started (novox/hq ADR 0135, issue 213).
+			if p, ok := resource.(*declaration.Process); ok && p.RunOnce {
+				gates = true
+			}
 			if gates {
 				failed.Gated = true
 				// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
@@ -607,6 +630,28 @@ func ApplyKeeping(
 			}
 			log(line)
 		}
+
+		// Its replacement applied: what it replaces goes now, once it is seen running (issue 213).
+		if waiting, has := handover[resource.Identity()]; has {
+			delete(handover, resource.Identity())
+			if err := handOver(ctx, run, resource, waiting, removeOrphan, &report, log); err != nil {
+				failures = append(failures, err)
+			}
+		}
+	}
+
+	// A replacement that did not apply — failed, skipped behind its module's step, held — leaves
+	// what it replaces running and recorded, said, for the next apply to hand over.
+	unhanded := make([]string, 0, len(handover))
+	for by := range handover {
+		unhanded = append(unhanded, by)
+	}
+	sort.Strings(unhanded)
+	for _, by := range unhanded {
+		for _, orphan := range handover[by] {
+			keptForReplacement(&report, log, orphan,
+				fmt.Sprintf("kept: %s, which replaces it, did not apply", by))
+		}
 	}
 
 	if !orphansRemoved {
@@ -1041,6 +1086,38 @@ type unitReloader interface {
 	ReloadUnits(ctx context.Context, run system.Runner) error
 }
 
+// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
+// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
+var serviceSettle = 2 * time.Second
+
+// stayedRunning is the state of a unit the host has just asked to run, read twice.
+//
+// **Because the first read races the failure.** A service manager returns when it has started the
+// process, and the unit is "activating" or "active" at that instant whatever the process is about
+// to do. A daemon that reads its configuration, refuses it and exits does so a fraction of a second
+// later — fail2ban took 221 milliseconds the day this was written — so a single read back says
+// running about a machine whose daemon is already gone, and the apply reports "restarted" for a
+// service that is dead. Every ban on both public machines was lost that way while every check
+// passed (novox/hq ADR 0184), which is the one shape of failure this host exists to refuse.
+//
+// So it looks again, after the moment in which that happens. It does not wait for a slow unit to
+// finish starting: a unit still coming up reads as running both times and is accepted, as before.
+// What this catches is a unit that was running and is not any more.
+func stayedRunning(ctx context.Context, sys system.System, run Runner, unit string) (string, error) {
+	state, err := sys.ServiceState(ctx, run, unit)
+	if err != nil || state != "running" {
+		return state, err
+	}
+	timer := time.NewTimer(serviceSettle)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return state, ctx.Err()
+	case <-timer.C:
+	}
+	return sys.ServiceState(ctx, run, unit)
+}
+
 func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
 	changed map[string]bool, previous store.Applied) (Outcome, error) {
 	if r.Stateless() {
@@ -1120,6 +1197,9 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 		// Read back. A service manager accepting a command says the transaction was accepted,
 		// not that the unit is running — one that starts and immediately dies satisfies it.
 		after, err := sys.ServiceState(ctx, run, r.Unit)
+		if err == nil && r.State == "running" {
+			after, err = stayedRunning(ctx, sys, run, r.Unit)
+		}
 		if err != nil {
 			return out, err
 		}
@@ -1140,7 +1220,7 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 		}
 		// Read back, for the same reason as above: a unit that starts and immediately dies
 		// satisfies a service manager and nothing else.
-		after, err := sys.ServiceState(ctx, run, r.Unit)
+		after, err := stayedRunning(ctx, sys, run, r.Unit)
 		if err != nil {
 			return out, err
 		}
@@ -1230,7 +1310,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service,
 	}
 	// Read back: it was running, and a restart or reload that left it otherwise is a failure —
 	// the machine's network manager down is not a change to report and move past.
-	after, err := sys.ServiceState(ctx, run, r.Unit)
+	after, err := stayedRunning(ctx, sys, run, r.Unit)
 	if err != nil {
 		return out, err
 	}
@@ -1407,6 +1487,25 @@ func applyPackage(ctx context.Context, sys system.System, r *declaration.Package
 	if err != nil {
 		return out, err
 	}
+	if r.Absent {
+		// Declared absent (novox/hq ADR 0180): removed when it is here, left alone when it is not.
+		if !installed {
+			out.Action = "unchanged"
+			out.Detail = "not installed, as declared"
+			return out, nil
+		}
+		if err := sys.RemovePackage(ctx, run, r.Package); err != nil {
+			return out, fmt.Errorf("removing %s: %w", r.Package, err)
+		}
+		if still, err := sys.PackageInstalled(ctx, run, r.Package); err != nil {
+			return out, err
+		} else if still {
+			return out, fmt.Errorf("%s was removed without error and the package database still has it", r.Package)
+		}
+		out.Action = "removed"
+		out.Detail = "declared absent; its configuration is left where the package manager leaves it"
+		return out, nil
+	}
 	if installed {
 		out.Action = "unchanged"
 		out.Detail = "already installed"
@@ -1588,6 +1687,10 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, c := range r.Capabilities {
 		b.WriteString("cap " + c + "\n")
 	}
+	// And where it logs (ADR 0179): the runtime cannot move a running container's output.
+	if r.Logging != "" {
+		b.WriteString("log " + r.Logging + "\n")
+	}
 	// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
 	// moves and the install is reported "updated" and re-established. Added only when present, so no
 	// ordinary container's or run-once step's digest moves for a field it does not set.
@@ -1785,6 +1888,11 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 	for _, c := range r.Capabilities {
 		args = append(args, "--cap-add", c)
 	}
+	if r.Logging != "" {
+		// The journal keeps the container's name on every line (CONTAINER_NAME), which is what a
+		// jail matches on (novox/hq ADR 0179); `docker logs` keeps working against the journal.
+		args = append(args, "--log-driver", r.Logging)
+	}
 	for _, d := range r.Dns {
 		args = append(args, "--dns", d)
 	}
@@ -2390,3 +2498,102 @@ func moduleOf(identity string) (string, bool) {
 	}
 	return identity[:at], true
 }
+
+// handOver removes what a process replaces, once the process is running and still is a moment later
+// (novox/hq issue 213, ADR 0184). A replacement that is not up keeps what it replaces in place
+// and recorded, and is this apply's failure: the old one answers until a later apply finds the new
+// one up.
+func handOver(ctx context.Context, run Runner, resource declaration.Resource,
+	waiting []store.Applied, removeOrphan func(store.Applied) error, report *Report,
+	log func(string)) *Error {
+	p, ok := resource.(*declaration.Process)
+	if !ok {
+		return nil
+	}
+	if why := stillUp(ctx, run, p.Name+".service"); why != "" {
+		for _, orphan := range waiting {
+			keptForReplacement(report, log, orphan,
+				fmt.Sprintf("kept: %s, which replaces it, is not running (%s)", p.ID, why))
+		}
+		return &Error{Resource: p.ID, Err: fmt.Errorf(
+			"%s was started and is not running a moment later (%s), so what it replaces was kept: %s",
+			p.Name, why, appliedIDs(waiting)), Done: *report}
+	}
+	for _, orphan := range waiting {
+		if err := removeOrphan(orphan); err != nil {
+			var failed *Error
+			if errors.As(err, &failed) {
+				return failed
+			}
+			return &Error{Resource: orphan.ID, Err: err, Done: *report}
+		}
+	}
+	return nil
+}
+
+// handoverSettle is how long a replacement must stay up before what it replaces goes. Longer than a
+// service's second look (serviceSettle): this one decides whether the last thing answering is
+// removed, and a process that fails on its store or its bus does so after it opened them, not in the
+// first instant. A test sets it to nothing.
+var handoverSettle = 10 * time.Second
+
+// stillUp says why a process's unit is not up and staying up, or nothing when it is: active and
+// running at two looks handoverSettle apart, the same main process both times, restarted by
+// nothing in between. Stricter than stayedRunning, which reads "activating" as running — the state
+// a crash-looping unit is in while it waits to be started again, which is exactly the replacement
+// that must not be handed anything.
+func stillUp(ctx context.Context, run Runner, unit string) string {
+	look := func() (map[string]string, string) {
+		out, err := run(ctx, "systemctl", "show", unit, "--property=ActiveState", "--property=SubState",
+			"--property=MainPID", "--property=NRestarts")
+		if err != nil {
+			return nil, err.Error()
+		}
+		got := map[string]string{}
+		for _, line := range strings.Split(out, "\n") {
+			if key, value, found := strings.Cut(strings.TrimSpace(line), "="); found {
+				got[key] = value
+			}
+		}
+		if got["ActiveState"] != "active" || got["SubState"] != "running" {
+			return got, fmt.Sprintf("%s/%s", got["ActiveState"], got["SubState"])
+		}
+		return got, ""
+	}
+	first, why := look()
+	if why != "" {
+		return why
+	}
+	timer := time.NewTimer(handoverSettle)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err().Error()
+	case <-timer.C:
+	}
+	second, why := look()
+	if why != "" {
+		return why
+	}
+	if first["MainPID"] != second["MainPID"] || first["NRestarts"] != second["NRestarts"] {
+		return fmt.Sprintf("restarted while it was watched (pid %s → %s, restarts %s → %s)",
+			first["MainPID"], second["MainPID"], first["NRestarts"], second["NRestarts"])
+	}
+	return ""
+}
+
+// keptForReplacement reports an orphan kept because what replaces it is not yet in its place.
+func keptForReplacement(report *Report, log func(string), orphan store.Applied, detail string) {
+	report.Outcomes = append(report.Outcomes, Outcome{
+		ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "kept", Detail: detail,
+	})
+	log(fmt.Sprintf("  kept %s (%s): %s", orphan.ID, orphan.Target, strings.TrimPrefix(detail, "kept: ")))
+}
+
+func appliedIDs(applied []store.Applied) string {
+	ids := make([]string, 0, len(applied))
+	for _, a := range applied {
+		ids = append(ids, a.ID)
+	}
+	return strings.Join(ids, ", ")
+}

internal/apply/archive.go

@@ -66,16 +66,10 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
 		}
 	}
 
-	if err := os.MkdirAll(r.Path, 0o755); err != nil {
-		return out, err
-	}
-	written, err := unpack(body, r.Path)
+	written, err := replaceWith(body, r.Path, r.Owner)
 	if err != nil {
 		return out, err
 	}
-	if err := ownAll(r.Path, r.Owner); err != nil {
-		return out, err
-	}
 	out.Action = "updated"
 	if previous.Wrote == "" {
 		out.Action = "created"
@@ -84,6 +78,63 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
 	return out, nil
 }
 
+// replaceWith makes the directory exactly the archive (novox/hq issue 220).
+//
+// **The tree on disk is the archive and nothing else.** The digest is the whole identity of what
+// is unpacked here, so a file the previous archive had and this one does not must go. Unpacked over
+// the old tree, it stayed: a bundle rebuilt as one file per entrypoint kept the package directory
+// of the version before, which code could still import, and a fix that removed a file worked on a
+// fresh machine only. So the archive is unpacked into a fresh directory beside the old one, owned,
+// and swapped in by rename. A running process keeps the files it has open, and the old tree is
+// removed only once the new one is in place. A failed unpack leaves the old tree untouched.
+func replaceWith(body []byte, path, owner string) (int, error) {
+	parent := filepath.Dir(path)
+	if err := os.MkdirAll(parent, 0o755); err != nil {
+		return 0, err
+	}
+	fresh := path + ".unpacking"
+	replaced := path + ".replaced"
+	// What an interrupted earlier attempt left beside the directory.
+	for _, leftover := range []string{fresh, replaced} {
+		if err := os.RemoveAll(leftover); err != nil {
+			return 0, err
+		}
+	}
+	if err := os.Mkdir(fresh, 0o755); err != nil {
+		return 0, err
+	}
+	written, err := unpack(body, fresh)
+	if err == nil {
+		err = ownAll(fresh, owner)
+	}
+	if err != nil {
+		os.RemoveAll(fresh)
+		return written, err
+	}
+	hadOne := true
+	if err := os.Rename(path, replaced); err != nil {
+		if !os.IsNotExist(err) {
+			os.RemoveAll(fresh)
+			return written, err
+		}
+		hadOne = false
+	}
+	if err := os.Rename(fresh, path); err != nil {
+		if hadOne {
+			// Put the old tree back rather than leave nothing at the path.
+			os.Rename(replaced, path)
+		}
+		os.RemoveAll(fresh)
+		return written, err
+	}
+	if hadOne {
+		if err := os.RemoveAll(replaced); err != nil {
+			return written, fmt.Errorf("%s is in place, and the tree it replaced could not be removed: %w", path, err)
+		}
+	}
+	return written, nil
+}
+
 func fetch(ctx context.Context, source string) ([]byte, error) {
 	request, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
 	if err != nil {

internal/apply/archive_replace_test.go

@@ -0,0 +1,68 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// novox/hq issue 220: the tree on disk is exactly the archive. A file the previous archive had and
+// this one does not is gone, and nothing is left beside the directory.
+func TestAnArchiveReplacesTheTreeItWasUnpackedOver(t *testing.T) {
+	dir := t.TempDir()
+	first, firstDigest := anArchive(t, map[string]string{"index.js": "old", "node_modules/dep/index.js": "dep"})
+	d := declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, first)+
+		`","digest":"`+firstDigest+`","path":"`+dir+`/tools"}`)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	second, secondDigest := anArchive(t, map[string]string{"index.js": "one file"})
+	d = declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, second)+
+		`","digest":"`+secondDigest+`","path":"`+dir+`/tools"}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "one file" {
+		t.Fatalf("index.js is %q", got)
+	}
+	if _, err := os.Stat(dir + "/tools/node_modules"); err == nil {
+		t.Fatal("the previous archive's directory is still there")
+	}
+	entries, _ := os.ReadDir(dir)
+	if len(entries) != 1 {
+		names := []string{}
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		t.Fatalf("beside the tree: %v", names)
+	}
+}
+
+// A tree is replaced only by one that unpacked whole: an archive refused halfway leaves the old
+// tree as it was.
+func TestARefusedArchiveLeavesTheTreeItWouldHaveReplaced(t *testing.T) {
+	dir := t.TempDir()
+	first, firstDigest := anArchive(t, map[string]string{"index.js": "old"})
+	d := declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, first)+
+		`","digest":"`+firstDigest+`","path":"`+dir+`/tools"}`)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bad, badDigest := anArchive(t, map[string]string{"index.js": "new", "../../escaped": "no"})
+	d = declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, bad)+
+		`","digest":"`+badDigest+`","path":"`+dir+`/tools"}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil); err == nil {
+		t.Fatal("an escaping archive was accepted")
+	}
+	if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "old" {
+		t.Fatalf("index.js is %q after a refused archive", got)
+	}
+	if entries, _ := os.ReadDir(dir); len(entries) != 1 {
+		t.Fatalf("%d entries beside the tree after a refused archive", len(entries))
+	}
+}

internal/apply/handover_test.go

@@ -0,0 +1,318 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// novox/hq issue 213: the controller moves from a container to a process on the one machine that
+// runs it. Every orphan is removed before anything is applied, so without a handover the container
+// went first and nothing answered the mesh's verbs while the process was fetched, unpacked and
+// started — and for ever, if it did not start. A process that `replaces` the container is applied
+// first; the container goes only once the process is running a moment later.
+
+// aMachine fakes the service manager and the container runtime: it records every command, answers
+// `systemctl show` with whether the process is running, and has a container until it is removed.
+type aMachine struct {
+	commands  []string
+	running   bool // what `systemctl show` says of the process once it was started
+	started   bool
+	container bool
+	timer     bool // whether a timer, once started, is up
+	crashing  bool // up at the first look, waiting to restart at the next
+	looks     int
+}
+
+func (m *aMachine) run(ctx context.Context, name string, args ...string) (string, error) {
+	line := name + " " + strings.Join(args, " ")
+	m.commands = append(m.commands, line)
+	switch {
+	case name == "systemctl" && len(args) > 0 && (args[0] == "restart" || args[0] == "start"):
+		m.started = true
+	case name == "systemctl" && len(args) > 0 && args[0] == "is-active" && strings.HasSuffix(args[len(args)-1], ".timer"):
+		if m.timer {
+			return "active", nil
+		}
+		return "inactive", errors.New("inactive")
+	case name == "systemctl" && len(args) > 0 && args[0] == "is-active":
+		if m.started && m.running {
+			return "active", nil
+		}
+		return "inactive", errors.New("inactive")
+	case name == "systemctl" && len(args) > 0 && args[0] == "show":
+		m.looks++
+		if m.crashing && m.started {
+			// Up at the first look; waiting to be started again, a new process, at the second.
+			if m.looks == 1 {
+				return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
+			}
+			return "ActiveState=activating\nSubState=auto-restart\nMainPID=0\nNRestarts=1\n", nil
+		}
+		if m.started && m.running {
+			return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
+		}
+		return "ActiveState=inactive\nSubState=dead\nMainPID=0\nNRestarts=0\n", nil
+	case name == "docker" && len(args) > 1 && args[0] == "rm":
+		m.container = false
+	case name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect":
+		if !m.container {
+			return "", errors.New("no such container")
+		}
+		return "true\t", nil
+	}
+	return "", nil
+}
+
+func (m *aMachine) index(prefix string) int {
+	for i, c := range m.commands {
+		if strings.HasPrefix(c, prefix) {
+			return i
+		}
+	}
+	return -1
+}
+
+// theController is a machine whose controller ran as a container, recorded, and a declaration that
+// runs it as a process from a bundle served here instead.
+func theController(t *testing.T, digest, source, extra string) (store.State, string) {
+	t.Helper()
+	known := store.State{Resources: []store.Applied{{
+		Origin: store.OriginDeclared, ID: "mesh-controller.server", Type: "container", Target: "mesh-controller",
+	}}}
+	return known, `{"declaration":1,"resources":[
+	  {"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + source +
+		`","digest":"` + digest + `","run":["./mesh-controller","serve"]` + extra + `}]}`
+}
+
+func onAMachine(t *testing.T) {
+	t.Helper()
+	serviceSettle, handoverSettle = 0, 0
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = t.TempDir(), t.TempDir()
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+}
+
+func TestAContainerAProcessReplacesGoesOnlyOnceTheProcessRuns(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{running: true, container: true}
+
+	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err != nil {
+		t.Fatalf("the handover failed: %v", err)
+	}
+	started, removed := m.index("systemctl restart mesh-controller.service"), m.index("docker rm -f mesh-controller")
+	if started < 0 || removed < 0 {
+		t.Fatalf("the process was not started or the container not removed: %v", m.commands)
+	}
+	if removed < started {
+		t.Fatalf("the container was removed before its replacement was started — a window with "+
+			"nothing answering: %v", m.commands)
+	}
+	if looked := m.index("systemctl show mesh-controller.service"); looked < 0 || looked > removed {
+		t.Errorf("the container was removed without looking whether the process runs: %v", m.commands)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
+		t.Errorf("the container's outcome is %+v, want removed", o)
+	}
+	if _, still := after.Find("mesh-controller.server"); still {
+		t.Error("the host still records the container it removed")
+	}
+	if _, has := after.Find("mesh-controller.controller"); !has {
+		t.Error("the process was not recorded")
+	}
+}
+
+// The case the handover exists for: the replacement does not stay up. The container keeps
+// answering, stays recorded so a later apply hands it over, and the apply says why it failed.
+func TestAContainerIsKeptWhenItsReplacementDoesNotRun(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{running: false, container: true}
+
+	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err == nil {
+		t.Fatal("a replacement that is not running was reported as a clean apply")
+	}
+	if !strings.Contains(err.Error(), "mesh-controller.server") {
+		t.Errorf("the failure does not name what was kept: %v", err)
+	}
+	if m.index("docker rm") >= 0 {
+		t.Fatalf("the container was removed though its replacement is not running: %v", m.commands)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
+		t.Errorf("the container's outcome is %+v, want kept", o)
+	}
+	if _, still := after.Find("mesh-controller.server"); !still {
+		t.Fatal("the container was forgotten, so no later apply would ever remove it")
+	}
+
+	// The next apply finds the process up and finishes the handover.
+	m.running, m.commands = true, nil
+	report, after, err = Apply(context.Background(), archHost(t), parse(t, raw), after,
+		store.OriginDeclared, m.run, nil, nil)
+	if err != nil {
+		t.Fatalf("the second apply failed: %v", err)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
+		t.Errorf("the second apply did not hand over: %+v (%v)", o, m.commands)
+	}
+	if _, still := after.Find("mesh-controller.server"); still {
+		t.Error("the container is still recorded after the handover")
+	}
+}
+
+// A replacement that never applied — its bundle is not what was declared — touches nothing, and
+// what it replaces keeps running.
+func TestAContainerIsKeptWhenItsReplacementFailsToApply(t *testing.T) {
+	onAMachine(t)
+	body, _ := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, "sha256:"+strings.Repeat("b", 64), serving(t, body),
+		`,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{running: true, container: true}
+
+	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err == nil {
+		t.Fatal("a replacement whose bundle did not match was reported applied")
+	}
+	if m.index("docker rm") >= 0 {
+		t.Fatalf("the container was removed though nothing replaced it: %v", m.commands)
+	}
+	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
+		t.Errorf("the container's outcome is %+v, want kept", o)
+	}
+	if _, still := after.Find("mesh-controller.server"); !still {
+		t.Fatal("the container was forgotten")
+	}
+}
+
+// Without `replaces` nothing changes: an orphan goes before anything is applied, as it always has.
+// Kept as a test because it is the window the field exists to close.
+func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), ``)
+	m := &aMachine{running: true, container: true}
+	if _, _, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if removed, started := m.index("docker rm -f mesh-controller"), m.index("systemctl restart mesh-controller.service"); removed < 0 || removed > started {
+		t.Fatalf("an orphan nothing replaces was not removed first: %v", m.commands)
+	}
+}
+
+// novox/hq issue 213, beyond what the oneshot unit (process_step_test.go) already holds: a step
+// written `./name` runs its own bundle's binary — the controller's preparation is its own binary —
+// and a step is started, never enabled.
+func TestAStepRunsItsOwnBundlesBinaryAndIsNotEnabled(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	m := &aMachine{running: true}
+	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
+	  "source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil); err != nil {
+		t.Fatalf("the step failed: %v", err)
+	}
+	for _, c := range m.commands {
+		if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") {
+			t.Errorf("the step was run directly or enabled: %v", m.commands)
+		}
+	}
+	unit, err := os.ReadFile(filepath.Join(unitDir, "mesh-controller-prepare.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare"
+	if !strings.Contains(string(unit), want) {
+		t.Errorf("the step does not run its own bundle's binary (%q):\n%s", want, unit)
+	}
+}
+
+func TestAStepThatFailsGatesItsModule(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	src := serving(t, body)
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if name == "systemctl" && len(args) > 1 && args[0] == "start" {
+			return "", errors.New("Job for mesh-controller-prepare.service failed")
+		}
+		if name == "systemctl" && len(args) > 0 && args[0] == "restart" {
+			t.Errorf("the module's process was started after its step failed")
+		}
+		return "", nil
+	}
+	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
+	  "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true},
+	  {"id":"mesh-controller.controller","type":"process","name":"mesh-controller",
+	  "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","serve"]}`)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
+	if err == nil {
+		t.Fatal("a failed step was reported as a clean apply")
+	}
+	if o := outcomeOf(report, "mesh-controller.controller"); o.Action != "skipped" {
+		t.Errorf("the process after a failed step was %+v, want skipped", o)
+	}
+}
+
+// A completed step is done: applied again unchanged, it is not run again — its service is never up
+// between runs, and reading that as "a daemon that stopped" re-ran the controller's preparation on
+// every apply. Nor is a scheduled run started off its cadence; its timer is what is kept up.
+func TestACompletedStepIsNotRunAgainAndAScheduleIsItsTimer(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"job": "#!/bin/sh\n"})
+	src := serving(t, body)
+	for _, mode := range []string{`"run-once":true`, `"schedule":"0 3 * * *"`} {
+		// The service of either is never up between runs; a scheduled one's timer is.
+		m := &aMachine{running: false, timer: true}
+		d := declare(t, `{"id":"m.job","type":"process","name":"m-job","source":"`+src+`","digest":"`+digest+
+			`","run":["./job"],`+mode+`}`)
+		_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
+		if err != nil {
+			t.Fatalf("%s: first apply: %v", mode, err)
+		}
+		m.commands = nil
+		report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, nil)
+		if err != nil {
+			t.Fatalf("%s: second apply: %v", mode, err)
+		}
+		if m.index("systemctl start m-job.service") >= 0 || m.index("systemctl restart m-job.service") >= 0 {
+			t.Errorf("%s: an unchanged apply ran the job again: %v", mode, m.commands)
+		}
+		if o := outcomeOf(report, "m.job"); o.Action != "unchanged" {
+			t.Errorf("%s: an unchanged apply reported %+v", mode, o)
+		}
+	}
+}
+
+// A replacement crash-looping between its restarts is not up, though the service manager calls it
+// "activating" — the state the host's ordinary second look accepts as running. Removing the
+// container on that reading would leave nothing answering.
+func TestAContainerIsKeptWhenItsReplacementIsCrashLooping(t *testing.T) {
+	onAMachine(t)
+	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
+	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
+	m := &aMachine{crashing: true, container: true}
+	_, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
+		store.OriginDeclared, m.run, nil, nil)
+	if err == nil || !strings.Contains(err.Error(), "auto-restart") {
+		t.Fatalf("a crash-looping replacement was accepted: %v", err)
+	}
+	if m.index("docker rm") >= 0 {
+		t.Fatalf("the container was removed for a replacement that keeps dying: %v", m.commands)
+	}
+	if _, still := after.Find("mesh-controller.server"); !still {
+		t.Fatal("the container was forgotten")
+	}
+}

internal/apply/left_out_test.go

@@ -173,3 +173,42 @@ func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
 		t.Fatal("a capability is not part of the container's spec")
 	}
 }
+
+// A package may be declared absent (novox/hq ADR 0180): removed when it is installed, read back,
+// left alone when it is not.
+func TestAPackageDeclaredAbsentIsRemovedWhenPresentAndLeftWhenNot(t *testing.T) {
+	installed := true
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if name != "pacman" {
+			return "", nil
+		}
+		switch args[0] {
+		case "-Q":
+			if args[1] == "pacman" || installed {
+				return args[1] + " 1.0\n", nil
+			}
+			return "", errors.New("package not found")
+		case "-R":
+			installed = false
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[{"id":"front-end","type":"package","package":"ufw","absent":true}]}`)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "removed" || !strings.Contains(strings.Join(ran, "\n"), "pacman -R --noconfirm ufw") {
+		t.Fatalf("an installed package declared absent was not removed: %+v\n%v", report.Outcomes[0], ran)
+	}
+	ran = nil
+	report, _, err = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "unchanged" || strings.Contains(strings.Join(ran, "\n"), "-R") {
+		t.Fatalf("a package already absent was touched: %+v\n%v", report.Outcomes[0], ran)
+	}
+}

internal/apply/logging_test.go

@@ -0,0 +1,43 @@
+package apply
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A container declared to log to the journal is run with the journal as its log driver, and the
+// place it logs is part of its spec, so moving it recreates the container (novox/hq ADR 0179).
+func TestAContainerLoggingToTheJournalIsRunThatWayAndRecreatedWhenMoved(t *testing.T) {
+	pinned := "postgres@sha256:" + strings.Repeat("a", 64)
+	var ran []string
+	run := func(_ context.Context, cmd string, args ...string) (string, error) {
+		if cmd == "docker" && len(args) > 0 && args[0] == "run" {
+			ran = args
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"front","type":"container","name":"front","image":"`+pinned+`","logging":"journald"}
+	]}`)
+	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	sent := false
+	for i, a := range ran {
+		if a == "--log-driver" && i+1 < len(ran) && ran[i+1] == "journald" {
+			sent = true
+		}
+	}
+	if !sent {
+		t.Fatalf("the container's output was not sent to the journal: %v", ran)
+	}
+	with := d.Resources[0].(*declaration.Container)
+	without := *with
+	without.Logging = ""
+	if containerSpec(with, inputs{}) == containerSpec(&without, inputs{}) {
+		t.Fatal("where a container logs is not part of its spec, so moving it would not recreate it")
+	}
+}

internal/apply/opening.go

@@ -76,6 +76,16 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri
 	if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive {
 		return "", nil
 	}
+	if !firewall.Installed(ctx, run) {
+		// Uninstalled (novox/hq ADR 0180): retired for good, by the module that replaced it. Said
+		// once, and nothing is asked of a command that is not there.
+		if rec.RetiredBy != firewall.RetiredRemoved {
+			rec.RetiredBy = firewall.RetiredRemoved
+			log("  the found firewall (ufw) is no longer installed; the mesh's filter is what filters this machine")
+			return "removed: ufw is no longer installed; the mesh's filter is what filters this machine", nil
+		}
+		return "", nil
+	}
 	active := firewall.Active(ctx, run)
 	if !active && !(rec.Forward != nil && !rec.DisabledByMesh) {
 		// Inactive, and either the mesh's doing already or nobody's recorded here: said as found,

internal/apply/opening_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
@@ -522,3 +523,33 @@ func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
 		t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall)
 	}
 }
+
+// A front end that is no longer installed is recorded as removed, said once, and asked nothing of
+// (novox/hq ADR 0180).
+func TestAnUninstalledFrontEndIsRetiredForGood(t *testing.T) {
+	dir := t.TempDir()
+	u := &ufwMachine{installed: false, ruleset: "table inet mesh\n"}
+	known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true,
+		RetiredBy: "mesh", FoundAt: time.Now()}}
+	converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
+	report, state, err := applyWith(t, converged, known, u.run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state.Firewall.RetiredBy != "removed" || !strings.Contains(report.Firewall, "no longer installed") {
+		t.Fatalf("record %+v, said %q", state.Firewall, report.Firewall)
+	}
+	u.asked = nil
+	report, _, err = applyWith(t, converged, state, u.run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Firewall != "" {
+		t.Errorf("said again: %q", report.Firewall)
+	}
+	for _, a := range u.asked {
+		if strings.HasPrefix(a, "ufw") && a != "ufw status" {
+			t.Errorf("asked something of a front end that is not there: %v", u.asked)
+		}
+	}
+}

internal/apply/process.go

@@ -77,11 +77,26 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		// Everything about it is as declared. Still asked whether it is RUNNING, because a
 		// declaration that is satisfied by a record rather than by the machine is how a stopped
 		// service reports success.
-		if active, err := run(ctx, "systemctl", "is-active", "--quiet", r.Name+".service"); err == nil {
+		// **The record is carried forward, not re-derived.** An unchanged outcome is recorded
+		// like any other, so one that said nothing about what was written erased the digest; the
+		// next apply then found no record, re-created the daemon, and the one after that found a
+		// record again — the node's runtime restarted every other cycle (novox/hq 04-ISSUES/210).
+		out.wrote = want
+		// **A step that ran is done, and a schedule is its timer** (novox/hq issue 213). Neither's
+		// service is meant to be up between runs, so asking whether it is — and starting it when it
+		// was not — ran a completed step again on every apply, and a scheduled run off its cadence.
+		if r.RunOnce {
+			return out, nil
+		}
+		unit := r.Name + ".service"
+		if r.Schedule != "" {
+			unit = r.Name + ".timer"
+		}
+		if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
 			_ = active
 			return out, nil
 		}
-		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
+		if _, err := run(ctx, "systemctl", "start", unit); err != nil {
 			return out, fmt.Errorf("%s is installed and would not start: %w", r.Name, err)
 		}
 		out.Action = "updated"
@@ -110,9 +125,22 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 	// so the machine is not asked to start something that needed a migration that did not happen.
 	// Nothing is left behind to ask afterwards: the record that it ran is the digest, which is why
 	// the identity above includes the command.
+	//
+	// **Run as the unit a daemon would be, once** (novox/hq design 38 WP4c). Run directly, the
+	// step started in the host's own working directory, without its environment, its environment
+	// files or its user — `node bootstrap/index.js` resolved from wherever the host ran and was told
+	// none of the words it was declared with. A oneshot unit carries all four exactly as a daemon's
+	// does, and starting one waits for it to finish and fails when it fails.
 	if r.RunOnce {
-		if _, err := run(ctx, r.Run[0], r.Run[1:]...); err != nil {
-			return out, fmt.Errorf("the %s step did not complete: %w", r.Name, err)
+		unit := filepath.Join(unitDir, r.Name+".service")
+		if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
+			return out, err
+		}
+		if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
+			return out, err
+		}
+		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
+			return out, fmt.Errorf("the %s step did not complete (journalctl -u %s.service says why): %w", r.Name, r.Name, err)
 		}
 		out.Action = "created"
 		if previous.Wrote != "" {
@@ -209,9 +237,9 @@ func unitFor(r *declaration.Process) string {
 	if r.User != "" {
 		fmt.Fprintf(&b, "User=%s\n", r.User)
 	}
-	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(r.Run, " "))
-	if r.Schedule != "" {
-		// Started by its timer and expected to finish. Restarting it would have it run
+	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
+	if r.Schedule != "" || r.RunOnce {
+		// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
 		// continuously between fires, which is the opposite of a schedule.
 		b.WriteString("Type=oneshot\n")
 		b.WriteString("\n")
@@ -341,3 +369,15 @@ func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, st
 	}
 	return "removed", "stopped; its unit and its bundle removed — the mesh's own code", nil
 }
+
+// runFrom is the command as the unit runs it. **A command written `./name` is that file in the
+// process's own unpacked bundle** (novox/hq ADR 0193): a bundle compiled to a binary runs itself,
+// and only the host knows where it unpacked it, while the service manager takes an absolute path or
+// a name it finds on its own search path — never one relative to the working directory.
+func runFrom(r *declaration.Process) []string {
+	run := append([]string(nil), r.Run...)
+	if len(run) > 0 && strings.HasPrefix(run[0], "./") {
+		run[0] = filepath.Join(daemonRoot, r.Name, strings.TrimPrefix(run[0], "./"))
+	}
+	return run
+}

internal/apply/process_step_test.go

@@ -0,0 +1,81 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A run-once process is a step run where, how and as whom it was declared (novox/hq design 38
+// WP4c): its bundle's directory, its environment and environment files, its user. Run directly,
+// it started in the host's own directory with none of them.
+func TestARunOnceProcessRunsAsItsOneshotUnit(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+
+	me, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, digest := anArchive(t, map[string]string{"bootstrap/index.js": "console.log(1)\n"})
+	var commands []string
+	fail := false
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		commands = append(commands, name+" "+strings.Join(args, " "))
+		if fail && name == "systemctl" && len(args) > 0 && args[0] == "start" {
+			return "", errors.New("exit status 1")
+		}
+		return "", nil
+	}
+	d := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"user":"`+me.Username+`",`+
+		`"env":{"MESH_ADMIN":"mesh-admin"},"env-file":["/var/lib/mesh/mosquitto/bootstrap.env"]}`)
+
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	unit, err := os.ReadFile(filepath.Join(units, "mosquitto-bootstrap.service"))
+	if err != nil {
+		t.Fatalf("no unit was written for the step: %v", err)
+	}
+	for _, want := range []string{
+		"WorkingDirectory=" + filepath.Join(bundles, "mosquitto-bootstrap"),
+		"EnvironmentFile=/var/lib/mesh/mosquitto/bootstrap.env",
+		`Environment="MESH_ADMIN=mesh-admin"`,
+		"User=" + me.Username,
+		"Type=oneshot",
+		"ExecStart=node bootstrap/index.js",
+	} {
+		if !strings.Contains(string(unit), want) {
+			t.Errorf("the step's unit lacks %q:\n%s", want, unit)
+		}
+	}
+	for _, never := range []string{"Restart=always", "[Install]", "Type=simple"} {
+		if strings.Contains(string(unit), never) {
+			t.Errorf("a step's unit says %q:\n%s", never, unit)
+		}
+	}
+	joined := strings.Join(commands, "; ")
+	if !strings.Contains(joined, "systemctl start mosquitto-bootstrap.service") {
+		t.Errorf("the step was not started as its unit: %s", joined)
+	}
+	if strings.Contains(joined, "node bootstrap/index.js") || strings.Contains(joined, "enable mosquitto-bootstrap") {
+		t.Errorf("the step was run directly or enabled: %s", joined)
+	}
+
+	// A step that fails fails the apply, and is not recorded as done.
+	fail = true
+	d2 := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"env":{"MESH_ADMIN":"changed"}}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, run, nil, nil); err == nil {
+		t.Error("a step that failed did not fail the apply")
+	}
+}

internal/apply/process_test.go

@@ -256,3 +256,73 @@ func TestAProcessRecordedUnderAPathlikeNameIsRefusedNotRemoved(t *testing.T) {
 		}
 	}
 }
+
+// novox/hq 04-ISSUES/210: the node's runtime was re-created — and restarted — on every reconcile,
+// because the host did not find what it wrote for a process the cycle before. Applying the same
+// process declaration twice must do no work the second time.
+func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+
+	body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"})
+	var commands []string
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		commands = append(commands, name+" "+strings.Join(args, " "))
+		return "", nil
+	}
+	d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
+
+	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
+		t.Fatalf("first apply: %+v, want created", o)
+	}
+	rec, ok := state.Find("node-tools.runtime")
+	if !ok || rec.Wrote == "" {
+		t.Fatalf("the host did not record what it wrote for the process: %+v", rec)
+	}
+	commands = nil
+	again, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(again, "node-tools.runtime"); o.Action != "unchanged" {
+		t.Errorf("second apply: %+v, want unchanged", o)
+	}
+	for _, c := range commands {
+		if strings.Contains(c, "restart") {
+			t.Errorf("the second apply restarted the process: %v", commands)
+		}
+	}
+	// And the record survives an unchanged apply: the third cycle is unchanged too. This is the
+	// cycle the live mesh showed — created, unchanged, created — before the record was carried.
+	if rec, _ := state.Find("node-tools.runtime"); rec.Wrote == "" {
+		t.Fatalf("an unchanged apply dropped the digest from the record: %+v", rec)
+	}
+	commands = nil
+	third, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(third, "node-tools.runtime"); o.Action != "unchanged" {
+		t.Errorf("third apply: %+v, want unchanged", o)
+	}
+}
+
+// novox/hq ADR 0193: a bundle compiled to a binary runs itself — `./name` is that file in the
+// process's own unpacked bundle, made absolute because the service manager takes nothing relative.
+func TestAProcessRunsItsOwnBundlesBinary(t *testing.T) {
+	unit := unitFor(&declaration.Process{Name: "node-tools", Run: []string{"./node-tools", "serve"}})
+	if !strings.Contains(unit, "ExecStart="+filepath.Join(daemonRoot, "node-tools", "node-tools")+" serve\n") {
+		t.Errorf("the binary is not run from its bundle:\n%s", unit)
+	}
+	other := unitFor(&declaration.Process{Name: "x", Run: []string{"node", "src/main.js"}})
+	if !strings.Contains(other, "ExecStart=node src/main.js\n") {
+		t.Errorf("a command found on the path was changed:\n%s", other)
+	}
+}

internal/apply/stayed_running_test.go

@@ -0,0 +1,100 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A daemon that reads a configuration it refuses dies a fraction of a second after the service
+// manager has reported it started — fail2ban took 221 milliseconds on the control node the day this
+// was written. One read back catches nothing: the unit is active at that instant. The mesh reported
+// the service "restarted" while every ban on two public machines was gone, and every check passed
+// (novox/hq ADR 0184). The host looks again, after the moment in which that happens.
+func TestAServiceThatDiesJustAfterItsRestartIsNotReportedRestarted(t *testing.T) {
+	serviceSettle = 0
+	// Alive at the first look after starting, dead at the second — the shape of a daemon that
+	// refuses what it was just given.
+	started, looks := false, 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			state := "active"
+			if started {
+				if looks++; looks >= 2 {
+					state = "failed"
+				}
+			}
+			return "LoadState=loaded\nActiveState=" + state + "\n", nil
+		}
+		if args[0] == "start" {
+			started = true
+		}
+		return "", nil
+	}
+	dir := t.TempDir()
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"conf","type":"file","path":"`+dir+`/jail.conf","content":"[sshd]\n","mode":"0644"},
+	  {"id":"run","type":"service","unit":"fail2ban.service","state":"running","restart-on":["conf"]}
+	]}`)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err == nil {
+		t.Fatal("a service that died just after being restarted was reported as restarted")
+	}
+	if !strings.Contains(err.Error(), "fail2ban.service") || !strings.Contains(err.Error(), "is stopped") {
+		t.Errorf("the failure does not name the unit and what it is now: %v", err)
+	}
+	if looks < 2 {
+		t.Errorf("the host looked at the unit %d time(s) after starting it; it must look again", looks)
+	}
+}
+
+// A unit still coming up reads as running at both looks and is accepted: the second look is for a
+// unit that WAS running and is not any more, never a wait for a slow one to finish starting.
+func TestAUnitStillStartingIsNotAFailure(t *testing.T) {
+	serviceSettle = 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			return "LoadState=loaded\nActiveState=activating\n", nil
+		}
+		return "", nil
+	}
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"s","type":"service","unit":"slow.service","state":"running"}
+	]}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("a unit still starting was reported as a failure: %v", err)
+	}
+}
+
+// And a service the declaration asks to be stopped is not waited on at all.
+func TestAServiceAskedToStopIsNotWaitedOn(t *testing.T) {
+	serviceSettle = 0
+	shows := 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			shows++
+			if shows == 1 {
+				return "LoadState=loaded\nActiveState=active\n", nil
+			}
+			return "LoadState=loaded\nActiveState=inactive\n", nil
+		}
+		return "", nil
+	}
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"s","type":"service","unit":"off.service","state":"stopped"}
+	]}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("stopping a service was reported as a failure: %v", err)
+	}
+}
+
+// The settle between a unit's two read-backs is a real pause on a machine and nothing in a test:
+// no test here drives a service manager that takes time, so paying it would only slow the suite
+// (novox/hq ADR 0184).
+func TestMain(m *testing.M) {
+	serviceSettle = 0
+	os.Exit(m.Run())
+}

internal/declaration/declaration.go

@@ -569,6 +569,18 @@ type Process struct {
 	// that runs once does not run on a schedule, and something that is not running cannot be
 	// restarted when a file changes.
 	Schedule string `json:"schedule,omitempty"`
+
+	// Replaces names resources this declaration no longer declares that this process takes the
+	// place of (novox/hq issue 213). Such a resource is not removed with the other orphans, before
+	// anything is applied: it is removed only once this process is applied and still running a
+	// moment later, and kept when it is not. So the thing being replaced answers until the thing
+	// replacing it does — the controller moving from its container to a process is the case: the
+	// container removed first left nothing answering the mesh's verbs for as long as fetching,
+	// unpacking and starting the process took, and for ever if the process did not start.
+	//
+	// For a process that stays up; a step or a scheduled run is not running a moment later by
+	// design, so there is nothing to hand over to.
+	Replaces []string `json:"replaces,omitempty"`
 }
 
 func (d *Process) Identity() string { return d.ID }
@@ -654,6 +666,20 @@ func (d *Process) validate(where string, _ bool) []string {
 			problems = append(problems, where+": "+err.Error())
 		}
 	}
+	if len(d.Replaces) > 0 && (d.RunOnce || d.Schedule != "") {
+		problems = append(problems, where+
+			": only a process that stays up replaces something — a step or a scheduled run is not "+
+			"running a moment later, so what it replaced would be removed with nothing in its place, "+
+			"or never")
+	}
+	for _, id := range d.Replaces {
+		switch {
+		case strings.TrimSpace(id) == "":
+			problems = append(problems, where+": replaces names an empty id")
+		case id == d.ID:
+			problems = append(problems, where+": a process cannot replace itself")
+		}
+	}
 	return problems
 }
 
@@ -870,6 +896,12 @@ type Package struct {
 	ID      string `json:"id"`
 	Type    Type   `json:"type"`
 	Package string `json:"package"`
+	// Absent declares that the package is NOT installed (novox/hq ADR 0180): the host removes it
+	// when it is, and leaves a machine that never had it alone. For the one case a module replaces
+	// software the machine was found with and the operator has decided it does not come back — the
+	// firewall front end a converged machine's filter module retired. Nothing to undo when the
+	// declaration drops it: the host does not install what a declaration stopped saying is absent.
+	Absent bool `json:"absent,omitempty"`
 }
 
 func (p *Package) Identity() string { return p.ID }
@@ -946,6 +978,14 @@ type Container struct {
 	// container; a privileged container stays undeclarable.
 	Capabilities []string `json:"capabilities,omitempty"`
 
+	// Logging names where the runtime sends this container's output: "journald" sends it to the
+	// machine's journal, under the container's name, where what reads the machine's logs — its
+	// intrusion prevention first of all (novox/hq ADR 0179) — can read it the way it reads the
+	// machine's own services. Empty keeps the runtime's default, which is a file of the runtime's
+	// own that nothing but the runtime reads. Part of the spec: a container that logs elsewhere
+	// is a different container, and the runtime cannot change a running one's driver.
+	Logging string `json:"logging,omitempty"`
+
 	// Networks are networks this container also joins once created, by name — a found network a
 	// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
 	// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
@@ -1051,6 +1091,10 @@ func (c *Container) validate(where string, _ bool) []string {
 				"capability's name (CAP_NET_ADMIN or NET_ADMIN)")
 		}
 	}
+	if c.Logging != "" && c.Logging != "journald" {
+		problems = append(problems, where+": logging is "+strconv.Quote(c.Logging)+", and the only place a "+
+			"container's output can be sent besides the runtime's own file is \"journald\"")
+	}
 	for _, n := range c.Networks {
 		problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
 		if n == c.Network {
@@ -1439,6 +1483,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 		d.Resources = append(d.Resources, resource)
 	}
 	problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
+	problems = append(problems, checkReplaces(d.Resources)...)
 	if env.Adoption == nil {
 		for _, r := range d.Resources {
 			if r.Kind() == TypeOpening {
@@ -1622,3 +1667,36 @@ func stripComments(raw []byte) []byte {
 	}
 	return []byte(strings.Join(kept, "\n"))
 }
+
+// checkReplaces refuses a `replaces` naming something this same declaration still declares, or
+// named by two processes (novox/hq issue 213). What is replaced is what the declaration no longer
+// says — a resource still declared is applied, not handed over, and one handed to two replacements
+// would go when the first of them came up, whatever became of the second.
+func checkReplaces(resources []Resource) []string {
+	declared := map[string]bool{}
+	for _, r := range resources {
+		declared[r.Identity()] = true
+	}
+	var problems []string
+	by := map[string]string{}
+	for _, r := range resources {
+		p, ok := r.(*Process)
+		if !ok {
+			continue
+		}
+		for _, id := range p.Replaces {
+			if declared[id] {
+				problems = append(problems, fmt.Sprintf(
+					"resource %q: replaces %q, which this declaration still declares — a process "+
+						"replaces what the declaration no longer says", p.ID, id))
+			}
+			if other, taken := by[id]; taken && other != p.ID {
+				problems = append(problems, fmt.Sprintf(
+					"resource %q: replaces %q, which %q replaces too — one thing has one replacement",
+					p.ID, id, other))
+			}
+			by[id] = p.ID
+		}
+	}
+	return problems
+}

internal/declaration/declaration_test.go

@@ -524,3 +524,24 @@ func TestACapabilityIsNamedOrRefused(t *testing.T) {
 		}
 	}
 }
+
+// A container may send its output to the machine's journal, and nowhere else but the runtime's own
+// file (novox/hq ADR 0179): what reads the machine's logs then reads the container's too.
+func TestAContainerMayLogToTheJournalAndNowhereElse(t *testing.T) {
+	image := "postgres@sha256:" + strings.Repeat("a", 64)
+	d, err := Parse([]byte(`{"declaration":1,"resources":[
+	  {"id":"front","type":"container","name":"front","image":"` + image + `","logging":"journald"}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := d.Resources[0].(*Container).Logging; got != "journald" {
+		t.Fatalf("logging read as %q", got)
+	}
+	for _, bad := range []string{`"syslog"`, `"none"`, `"json-file"`} {
+		if _, err := Parse([]byte(`{"declaration":1,"resources":[
+		  {"id":"front","type":"container","name":"front","image":"` + image + `","logging":` + bad + `}]}`)); err == nil {
+			t.Errorf("%s was accepted as a place to log", bad)
+		}
+	}
+}

internal/declaration/replaces_test.go

@@ -0,0 +1,63 @@
+package declaration
+
+import (
+	"strings"
+	"testing"
+)
+
+// novox/hq issue 213: a process says what it takes the place of, so the host can keep the old one
+// answering until the new one does. What it may name is narrow, and each refusal is said here.
+
+const aReplacingProcess = `{"id":"m.controller","type":"process","name":"m","source":"https://store.invalid/m",
+  "digest":"sha256:` + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + `","run":["./m","serve"]%s}`
+
+func replacing(extra string) string {
+	return `{"declaration":1,"resources":[` + strings.Replace(aReplacingProcess, "%s", extra, 1) + `]}`
+}
+
+func TestAProcessMayNameWhatItReplaces(t *testing.T) {
+	d, err := Parse([]byte(replacing(`,"replaces":["m.server"]`)))
+	if err != nil {
+		t.Fatalf("a process replacing an undeclared container was refused: %v", err)
+	}
+	if p := d.Resources[0].(*Process); len(p.Replaces) != 1 || p.Replaces[0] != "m.server" {
+		t.Fatalf("replaces was not read: %+v", p)
+	}
+}
+
+// What is replaced is what the declaration no longer says. A resource still declared is applied,
+// and handing it over as well would remove something the same declaration asks to keep.
+func TestAProcessMayNotReplaceSomethingStillDeclared(t *testing.T) {
+	raw := `{"declaration":1,"resources":[
+	  {"id":"m.server","type":"directory","path":"/tmp/x"},
+	  ` + strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + `]}`
+	if _, err := Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "still declares") {
+		t.Fatalf("a process replacing a declared resource was accepted: %v", err)
+	}
+}
+
+func TestOnlyAProcessThatStaysUpReplacesAnything(t *testing.T) {
+	for _, mode := range []string{`,"run-once":true`, `,"schedule":"0 3 * * *"`} {
+		if _, err := Parse([]byte(replacing(mode + `,"replaces":["m.server"]`))); err == nil {
+			t.Errorf("a process with %s was allowed to replace something", mode)
+		}
+	}
+}
+
+func TestAProcessCannotReplaceItselfOrNothing(t *testing.T) {
+	for _, bad := range []string{`,"replaces":["m.controller"]`, `,"replaces":[""]`} {
+		if _, err := Parse([]byte(replacing(bad))); err == nil {
+			t.Errorf("replaces %s was accepted", bad)
+		}
+	}
+}
+
+func TestOneThingHasOneReplacement(t *testing.T) {
+	second := strings.NewReplacer(`"id":"m.controller"`, `"id":"m.other"`, `"name":"m"`, `"name":"other"`).
+		Replace(strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1))
+	raw := `{"declaration":1,"resources":[` +
+		strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + "," + second + `]}`
+	if _, err := Parse([]byte(raw)); err == nil {
+		t.Fatal("two processes replacing one resource were accepted")
+	}
+}

internal/firewall/filters.go

@@ -179,27 +179,18 @@ func legacyFilters(rules, tool string, ufwActive bool) []Filter {
 			}
 		}
 	}
-	var entered func(chain string, seen map[string]bool) bool
-	entered = func(chain string, seen map[string]bool) bool {
-		if seen[chain] || accepting[chain] || len(jumpedFrom[chain]) == 0 {
-			return false
-		}
-		seen[chain] = true
-		for _, from := range jumpedFrom[chain] {
-			if p, builtIn := policy[from]; builtIn {
-				if p != "ACCEPT" {
-					return false
-				}
-				continue
-			}
-			if !entered(from, seen) {
-				return false
-			}
-		}
-		return true
-	}
+	// A chain of refusals is a ban list when every refusal names the sources it refuses and the
+	// chain accepts nothing — the same rule the nftables side applies, and no more.
+	//
+	// **The policy of the chains that jump to it says nothing about what it is.** An earlier cut
+	// required every path into the chain to come from a built-in whose policy accepts, and the
+	// mesh's own intrusion prevention then read as a foreign rule set on the home server: its ban
+	// chain hangs off the container runtime's user chain as well as INPUT, and that machine's
+	// forward policy is DROP because the runtime set it. The machine reported "NOT the mesh alone"
+	// about a chain the mesh had just written (novox/hq ADR 0186). The policy is already classified
+	// where it belongs — as the runtime's — so requiring it here counted it twice.
 	ban := func(chain, line string) bool {
-		return bansSources(line) && entered(chain, map[string]bool{})
+		return bansSources(line) && !accepting[chain] && len(jumpedFrom[chain]) > 0
 	}
 	type seen struct {
 		owner string
@@ -319,8 +310,17 @@ func Active(ctx context.Context, run Runner) bool {
 	return err == nil && statusActive(out)
 }
 
+// Installed says whether ufw is on this machine at all: a command that is not there is a front end
+// that was uninstalled (novox/hq ADR 0180), not one that is silent.
+func Installed(ctx context.Context, run Runner) bool {
+	_, err := run(ctx, "ufw", "status")
+	return !missing(err)
+}
+
 // Retirements of a found firewall, as the host records them.
 const (
 	RetiredByMesh  = "mesh"
 	RetiredFoundSo = "found-inactive"
+	// RetiredRemoved is a front end uninstalled by the module that replaced it (ADR 0180).
+	RetiredRemoved = "removed"
 )

internal/firewall/neighbour_ban_test.go

@@ -0,0 +1,60 @@
+package firewall
+
+import (
+	"os"
+	"testing"
+)
+
+// The mesh's own ban list is a ban wherever it hangs (novox/hq ADR 0186).
+//
+// Captured from the home server after the intrusion prevention had banned four addresses: its ban
+// chain is jumped to from INPUT, whose policy accepts, and from the container runtime's user chain,
+// which hangs off a FORWARD the runtime set to DROP. Requiring every path to come from an accepting
+// built-in made the machine report "NOT the mesh alone" about a chain the mesh had just written.
+func TestTheMeshsOwnBanChainIsABanBehindADroppingForward(t *testing.T) {
+	legacy, err := os.ReadFile("testdata/home-server-bans-S.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	filters := Filters("", map[string]string{"iptables-legacy": string(legacy)}, false)
+
+	var ban, other []string
+	for _, f := range filters {
+		switch f.Owner {
+		case OwnerBan:
+			ban = append(ban, f.Where)
+		case OwnerOther:
+			other = append(other, f.Where)
+		}
+	}
+	if len(other) > 0 {
+		t.Errorf("the machine reports %v as rule sets the mesh did not write", other)
+	}
+	found := false
+	for _, w := range ban {
+		if w == "chain f2b-route-proxy (iptables-legacy)" {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("the intrusion prevention's own chain was not read as a ban; bans were %v", ban)
+	}
+	if !Alone(filters) {
+		t.Error("a machine filtered by the mesh and its own bans does not read as the mesh alone")
+	}
+}
+
+// A chain that accepts anything is doing more than banning, and is still not a ban — which is what
+// keeps a predecessor's allow-and-drop chain classified as something the operator must look at.
+func TestAChainThatAcceptsIsNotABan(t *testing.T) {
+	rules := "-P INPUT ACCEPT\n" +
+		"-A INPUT -j HAL-MESH-ONLY\n" +
+		"-N HAL-MESH-ONLY\n" +
+		"-A HAL-MESH-ONLY -s 10.0.0.0/8 -j ACCEPT\n" +
+		"-A HAL-MESH-ONLY -s 203.0.113.7/32 -j DROP\n"
+	for _, f := range Filters("", map[string]string{"iptables-legacy": rules}, false) {
+		if f.Where == "chain HAL-MESH-ONLY (iptables-legacy)" && f.Owner != OwnerOther {
+			t.Errorf("a chain that accepts was classified as %s", f.Owner)
+		}
+	}
+}

internal/firewall/testdata/home-server-bans-S.txt

@@ -0,0 +1,147 @@
+-P INPUT ACCEPT
+-P FORWARD DROP
+-P OUTPUT ACCEPT
+-N DOCKER
+-N DOCKER-BRIDGE
+-N DOCKER-CT
+-N DOCKER-FORWARD
+-N DOCKER-INTERNAL
+-N DOCKER-USER
+-N f2b-route-proxy
+-N ufw-after-forward
+-N ufw-after-input
+-N ufw-after-logging-forward
+-N ufw-after-logging-input
+-N ufw-after-logging-output
+-N ufw-after-output
+-N ufw-before-forward
+-N ufw-before-input
+-N ufw-before-logging-forward
+-N ufw-before-logging-input
+-N ufw-before-logging-output
+-N ufw-before-output
+-N ufw-reject-forward
+-N ufw-reject-input
+-N ufw-reject-output
+-N ufw-track-forward
+-N ufw-track-input
+-N ufw-track-output
+-A INPUT -p tcp -j f2b-route-proxy
+-A INPUT -j ufw-before-logging-input
+-A INPUT -j ufw-before-input
+-A INPUT -j ufw-after-input
+-A INPUT -j ufw-after-logging-input
+-A INPUT -j ufw-reject-input
+-A INPUT -j ufw-track-input
+-A FORWARD -j DOCKER-USER
+-A FORWARD -j DOCKER-FORWARD
+-A FORWARD -j ufw-before-logging-forward
+-A FORWARD -j ufw-before-forward
+-A FORWARD -j ufw-after-forward
+-A FORWARD -j ufw-after-logging-forward
+-A FORWARD -j ufw-reject-forward
+-A FORWARD -j ufw-track-forward
+-A OUTPUT -j ufw-before-logging-output
+-A OUTPUT -j ufw-before-output
+-A OUTPUT -j ufw-after-output
+-A OUTPUT -j ufw-after-logging-output
+-A OUTPUT -j ufw-reject-output
+-A OUTPUT -j ufw-track-output
+-A DOCKER -d 172.17.0.18/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8686 -j ACCEPT
+-A DOCKER -d 172.17.0.14/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8989 -j ACCEPT
+-A DOCKER -d 172.17.0.15/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 7878 -j ACCEPT
+-A DOCKER -d 172.17.0.5/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9117 -j ACCEPT
+-A DOCKER -d 172.17.0.13/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6789 -j ACCEPT
+-A DOCKER -d 172.19.0.2/32 ! -i br-32062158f584 -o br-32062158f584 -p tcp -m tcp --dport 8080 -j ACCEPT
+-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 5432 -j ACCEPT
+-A DOCKER -d 172.27.0.2/32 ! -i br-0910a98c6158 -o br-0910a98c6158 -p tcp -m tcp --dport 5678 -j ACCEPT
+-A DOCKER -d 172.17.0.21/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3579 -j ACCEPT
+-A DOCKER -d 172.17.0.19/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8181 -j ACCEPT
+-A DOCKER -d 172.17.0.17/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8787 -j ACCEPT
+-A DOCKER -d 172.17.0.16/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6767 -j ACCEPT
+-A DOCKER -d 172.17.0.12/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
+-A DOCKER -d 172.17.0.11/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT
+-A DOCKER -d 172.17.0.10/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9443 -j ACCEPT
+-A DOCKER -d 172.17.0.10/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9000 -j ACCEPT
+-A DOCKER -d 172.17.0.9/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
+-A DOCKER -d 172.17.0.7/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 1880 -j ACCEPT
+-A DOCKER -d 172.28.0.2/32 ! -i br-b11461b5b028 -o br-b11461b5b028 -p tcp -m tcp --dport 80 -j ACCEPT
+-A DOCKER -d 172.23.0.14/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 6543 -j ACCEPT
+-A DOCKER -d 172.23.0.14/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 5432 -j ACCEPT
+-A DOCKER -d 172.23.0.5/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 8000 -j ACCEPT
+-A DOCKER -d 172.26.0.3/32 ! -i br-b0fec361ccaa -o br-b0fec361ccaa -p tcp -m tcp --dport 6167 -j ACCEPT
+-A DOCKER -d 172.26.0.2/32 ! -i br-b0fec361ccaa -o br-b0fec361ccaa -p tcp -m tcp --dport 80 -j ACCEPT
+-A DOCKER -d 172.17.0.8/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8000 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10001 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8880 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8843 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8443 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8080 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6789 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 5514 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 3478 -j ACCEPT
+-A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 1900 -j ACCEPT
+-A DOCKER -d 172.25.0.3/32 ! -i br-b98821f7dc38 -o br-b98821f7dc38 -p tcp -m tcp --dport 8000 -j ACCEPT
+-A DOCKER -d 172.18.0.3/32 ! -i br-442a0bfc65f8 -o br-442a0bfc65f8 -p tcp -m tcp --dport 1433 -j ACCEPT
+-A DOCKER -d 172.20.0.3/32 ! -i br-afa37ac8b33d -o br-afa37ac8b33d -p tcp -m tcp --dport 8081 -j ACCEPT
+-A DOCKER -d 172.20.0.3/32 ! -i br-afa37ac8b33d -o br-afa37ac8b33d -p tcp -m tcp --dport 1883 -j ACCEPT
+-A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8086 -j ACCEPT
+-A DOCKER -d 172.21.0.2/32 ! -i br-df15d8e19ec7 -o br-df15d8e19ec7 -p tcp -m tcp --dport 6379 -j ACCEPT
+-A DOCKER -d 172.30.0.3/32 ! -i br-521eab9a3a5e -o br-521eab9a3a5e -p tcp -m tcp --dport 8283 -j ACCEPT
+-A DOCKER -d 172.17.0.3/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
+-A DOCKER ! -i br-32062158f584 -o br-32062158f584 -j DROP
+-A DOCKER ! -i docker0 -o docker0 -j DROP
+-A DOCKER ! -i br-521eab9a3a5e -o br-521eab9a3a5e -j DROP
+-A DOCKER ! -i br-df15d8e19ec7 -o br-df15d8e19ec7 -j DROP
+-A DOCKER ! -i br-afa37ac8b33d -o br-afa37ac8b33d -j DROP
+-A DOCKER ! -i br-442a0bfc65f8 -o br-442a0bfc65f8 -j DROP
+-A DOCKER ! -i br-b98821f7dc38 -o br-b98821f7dc38 -j DROP
+-A DOCKER ! -i br-b0fec361ccaa -o br-b0fec361ccaa -j DROP
+-A DOCKER ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -j DROP
+-A DOCKER ! -i br-b11461b5b028 -o br-b11461b5b028 -j DROP
+-A DOCKER ! -i br-2df4e541b877 -o br-2df4e541b877 -j DROP
+-A DOCKER ! -i br-0910a98c6158 -o br-0910a98c6158 -j DROP
+-A DOCKER-BRIDGE -o br-32062158f584 -j DOCKER
+-A DOCKER-BRIDGE -o docker0 -j DOCKER
+-A DOCKER-BRIDGE -o br-521eab9a3a5e -j DOCKER
+-A DOCKER-BRIDGE -o br-df15d8e19ec7 -j DOCKER
+-A DOCKER-BRIDGE -o br-afa37ac8b33d -j DOCKER
+-A DOCKER-BRIDGE -o br-442a0bfc65f8 -j DOCKER
+-A DOCKER-BRIDGE -o br-b98821f7dc38 -j DOCKER
+-A DOCKER-BRIDGE -o br-b0fec361ccaa -j DOCKER
+-A DOCKER-BRIDGE -o br-66ffa5c1cba5 -j DOCKER
+-A DOCKER-BRIDGE -o br-b11461b5b028 -j DOCKER
+-A DOCKER-BRIDGE -o br-2df4e541b877 -j DOCKER
+-A DOCKER-BRIDGE -o br-0910a98c6158 -j DOCKER
+-A DOCKER-CT -o br-32062158f584 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-521eab9a3a5e -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-df15d8e19ec7 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-afa37ac8b33d -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-442a0bfc65f8 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-b98821f7dc38 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-b0fec361ccaa -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-66ffa5c1cba5 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-b11461b5b028 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-2df4e541b877 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-CT -o br-0910a98c6158 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-FORWARD -j DOCKER-CT
+-A DOCKER-FORWARD -j DOCKER-INTERNAL
+-A DOCKER-FORWARD -j DOCKER-BRIDGE
+-A DOCKER-FORWARD -i br-32062158f584 -j ACCEPT
+-A DOCKER-FORWARD -i docker0 -j ACCEPT
+-A DOCKER-FORWARD -i br-521eab9a3a5e -j ACCEPT
+-A DOCKER-FORWARD -i br-df15d8e19ec7 -j ACCEPT
+-A DOCKER-FORWARD -i br-afa37ac8b33d -j ACCEPT
+-A DOCKER-FORWARD -i br-442a0bfc65f8 -j ACCEPT
+-A DOCKER-FORWARD -i br-b98821f7dc38 -j ACCEPT
+-A DOCKER-FORWARD -i br-b0fec361ccaa -j ACCEPT
+-A DOCKER-FORWARD -i br-66ffa5c1cba5 -j ACCEPT
+-A DOCKER-FORWARD -i br-b11461b5b028 -j ACCEPT
+-A DOCKER-FORWARD -i br-2df4e541b877 -j ACCEPT
+-A DOCKER-FORWARD -i br-0910a98c6158 -j ACCEPT
+-A DOCKER-USER -p tcp -j f2b-route-proxy
+-A f2b-route-proxy -s 13.70.107.184/32 -j REJECT --reject-with icmp-port-unreachable
+-A f2b-route-proxy -s 45.138.12.51/32 -j REJECT --reject-with icmp-port-unreachable
+-A f2b-route-proxy -s 20.214.191.94/32 -j REJECT --reject-with icmp-port-unreachable
+-A f2b-route-proxy -j RETURN

internal/system/alpine.go

@@ -46,6 +46,11 @@ func (a alpine) PackageInstalled(ctx context.Context, run Runner, name string) (
 	return strings.TrimSpace(out) != "", nil
 }
 
+func (alpine) RemovePackage(ctx context.Context, run Runner, name string) error {
+	_, err := run(ctx, "apk", "del", name)
+	return err
+}
+
 func (alpine) InstallPackage(ctx context.Context, run Runner, name string) error {
 	_, err := run(ctx, "apk", "add", "--no-cache", name)
 	return err

internal/system/android.go

@@ -65,6 +65,10 @@ func (a android) InstallPackage(context.Context, Runner, string) error {
 	return fmt.Errorf("%w: package", ErrUnsupported)
 }
 
+func (a android) RemovePackage(context.Context, Runner, string) error {
+	return fmt.Errorf("%w: package", ErrUnsupported)
+}
+
 func (a android) ServiceState(context.Context, Runner, string) (string, error) {
 	return "", fmt.Errorf("%w: service (init is not reachable without root)", ErrUnsupported)
 }

internal/system/arch.go

@@ -3,7 +3,10 @@ package system
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 )
@@ -39,6 +42,14 @@ func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bo
 	return true, nil
 }
 
+// RemovePackage removes one package and nothing it depends on: `-R`, not `-Rs`, because what else
+// relied on a dependency is not this declaration's to know. pacman keeps a configuration file the
+// operator changed as `.pacsave`, which is what "never flushed" comes to once the front end is gone.
+func (arch) RemovePackage(ctx context.Context, run Runner, name string) error {
+	_, err := run(ctx, "pacman", "-R", "--noconfirm", name)
+	return err
+}
+
 func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name)
 	if err == nil {
@@ -48,42 +59,119 @@ func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	// **The package manager's own words, and a name for the case that looks like a bug in the
 	// declaration and is not.** A stale index asks the mirrors for a version they have already
 	// superseded and gets a 404 from every one of them — so the package exists, the declaration is
-	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002).
+	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
+	// old as the index fails one step later, on the signature of whatever a mirror still had.
+	//
+	// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
+	// the error rather than the output; this classifier read the output alone and so never saw a
+	// single "failed retrieving file", and the control node reported a ten-week-old database as
+	// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
 	//
 	// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
 	// libraries this machine does not have: a partial upgrade, which Arch does not support and
 	// which breaks the machine in a way that surfaces much later as something unrelated. The
 	// remedy is a full upgrade, and it is a decision about the whole machine rather than
-	// something to do silently in the middle of applying one resource.
-	//
-	// So this says which of the two it is looking at. A declaration that is wrong and a machine
-	// that is out of date fail identically otherwise, and they are fixed in completely different
-	// places.
-	if staleIndex(out) {
+	// something to do silently in the middle of applying one resource. Whose decision, and on
+	// what schedule, is issue 205's question; until it is answered the host says what it sees.
+	said := strings.TrimSpace(out + "\n" + err.Error())
+	switch classifyInstallFailure(said) {
+	case installStale:
 		return fmt.Errorf(
-			"%s could not be fetched from any mirror, which is what a stale package index looks "+
-				"like: this machine is asking for a version the mirrors have replaced. The "+
-				"package and the declaration are probably both fine. It is fixed by upgrading "+
-				"the machine, not by this host syncing one package — that would be a partial "+
-				"upgrade, which this distribution does not support.\n\n%s",
-			name, strings.TrimSpace(out))
+			"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
+				"the package database on this machine is %s and the mirrors no longer serve what it "+
+				"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
+				"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
+				"host does not sync one package by itself, because on this distribution that is a "+
+				"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
+			name, syncDatabaseAge(), name, said)
+	case installMirrors:
+		return fmt.Errorf(
+			"no mirror could be reached to fetch %s, and the package database on this machine is "+
+				"%s: this reads as the mirrors or the network, not as this machine being out of "+
+				"date — try again when they answer.\n\n%s",
+			name, syncDatabaseAge(), said)
 	}
 	return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
 }
 
-// staleIndex reports whether a failed install looks like the machine's view being old rather than
-// the package being wrong.
-//
-// By what the package manager said, because there is nothing else to go on: the exit code is the
-// same for both.
-func staleIndex(out string) bool {
-	said := strings.ToLower(out)
-	if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") {
-		return false
+// How a failed install is read, from what the package manager said.
+type installFailure int
+
+const (
+	installOther installFailure = iota
+	// installStale: the machine's package database or keyring is older than what the mirrors
+	// serve — every mirror 404s the file the database names, or a package that did arrive fails
+	// its signature against a keyring that never saw the key.
+	installStale
+	// installMirrors: no mirror could be reached at all, and nothing says the database is old.
+	installMirrors
+)
+
+// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
+// code is the same for every one of these.
+func classifyInstallFailure(said string) installFailure {
+	lower := strings.ToLower(said)
+	gone := strings.Count(lower, "returned error: 404")
+	fetching := strings.Contains(lower, "failed retrieving file")
+	badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
+		strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
+		strings.Contains(lower, "is unknown trust") ||
+		strings.Contains(lower, "could not be looked up remotely")
+	switch {
+	case badSignature:
+		return installStale
+	case fetching && gone > 0:
+		// Every mirror, not one: a single mirror failing is an ordinary transient thing and
+		// retrying is the answer. pacman walks its whole mirror list before giving up, so more
+		// than one 404 among the lines is the index being old rather than one host being wrong.
+		if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
+			!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
+			return installStale
+		}
+		return installMirrors
+	case fetching:
+		return installMirrors
 	}
-	// Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying
-	// is the answer; every one of them saying the file is gone is the index being old.
-	return strings.Contains(said, "error") || strings.Count(said, "404") > 1
+	return installOther
+}
+
+// staleIndex is the yes-or-no form older callers and tests use.
+func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
+
+// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
+// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
+// install so a ten-week-old database is told apart from a mirror outage by reading one line.
+//
+// A variable so a test can say what the machine's database looks like without having one.
+var syncDatabaseAge = func() string {
+	entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
+	if err != nil || len(entries) == 0 {
+		return "of unknown age (no sync database found under /var/lib/pacman/sync)"
+	}
+	var newest time.Time
+	for _, e := range entries {
+		info, err := os.Stat(e)
+		if err == nil && info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	if newest.IsZero() {
+		return "of unknown age"
+	}
+	return describeAge(newest, time.Now())
+}
+
+// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
+func describeAge(when, now time.Time) string {
+	days := int(now.Sub(when).Hours() / 24)
+	span := fmt.Sprintf("%d days old", days)
+	switch {
+	case days < 1:
+		span = "less than a day old"
+	case days >= 14:
+		span = fmt.Sprintf("%d weeks old", days/7)
+	}
+	return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
 }
 
 // ServiceState reads what systemd says about a unit.

internal/system/arch_install_test.go

@@ -0,0 +1,115 @@
+package system
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+// pacman's own words from the control node on 2026-10-02 (novox/hq 04-ISSUES/205): every mirror
+// 404s the versioned file a ten-week-old database names, and the one copy that arrives fails its
+// signature. Errors are pacman's stderr, which the runner folds into the error, not the output.
+const staleStderr = `error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.hetzner.com : The requested URL returned error: 404
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.rackspace.com : The requested URL returned error: 404
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from arch.lucassymons.net : Could not resolve host: arch.lucassymons.net
+warning: fatal error from arch.lucassymons.net, skipping for the remainder of this transaction
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirrors.cqu.edu.cn : Connection timed out after 10002 milliseconds
+error: nodejs: signature from "Bert Peters (packager key) <bertptrs@archlinux.org>" is invalid
+error: failed to commit transaction (invalid or corrupted package (PGP signature))`
+
+const staleStdout = `resolving dependencies...
+looking for conflicting packages...
+
+Packages (4) ada-3.4.4-1  c-ares-1.34.8-1  simdjson-1:4.6.4-1  nodejs-26.5.0-1
+
+:: Retrieving packages...
+ nodejs-26.5.0-1-x86_64 downloading...
+checking keyring...
+checking package integrity...
+:: File /var/cache/pacman/pkg/nodejs-26.5.0-1-x86_64.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)).
+Errors occurred, no packages were upgraded.`
+
+// A runner that behaves as ExecRunner does on failure: stdout as the output, stderr in the error.
+func pacmanFailing(stdout, stderr string) Runner {
+	return func(_ context.Context, name string, args ...string) (string, error) {
+		return stdout, errors.New(name + " exited 1: " + stderr)
+	}
+}
+
+func TestAStaleDatabaseIsSaidAsOneWithItsAgeAndTheRemedy(t *testing.T) {
+	was := syncDatabaseAge
+	defer func() { syncDatabaseAge = was }()
+	syncDatabaseAge = func() string {
+		return describeAge(time.Date(2026, 7, 24, 16, 56, 0, 0, time.UTC), time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC))
+	}
+
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing(staleStdout, staleStderr), "nodejs")
+	if err == nil {
+		t.Fatal("a failed install must fail")
+	}
+	for _, want := range []string{
+		"the package database on this machine is from 2026-07-24, 10 weeks old",
+		"stale package index",
+		"upgrading the machine — a full upgrade (`pacman -Syu`) by its operator — before the mesh can install nodejs",
+		"partial upgrade",
+		"returned error: 404", // pacman's own words follow
+	} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the error does not say %q:\n%s", want, err)
+		}
+	}
+	if strings.Contains(err.Error(), "mirrors or the network") {
+		t.Errorf("a stale database must not be read as a mirror outage:\n%s", err)
+	}
+}
+
+// The same 404s read from stderr alone — the half this classifier used to be blind to.
+func TestTheClassifierReadsWhatPacmanWroteToStderr(t *testing.T) {
+	if classifyInstallFailure(staleStderr) != installStale {
+		t.Fatal("every mirror 404ing the named file is a stale database")
+	}
+	if classifyInstallFailure(staleStdout) != installStale {
+		t.Fatal("a corrupted-signature line alone is a stale keyring")
+	}
+	if classifyInstallFailure("") != installOther {
+		t.Fatal("nothing said is nothing classified")
+	}
+}
+
+func TestAnUnreachableMirrorWithAFreshDatabaseIsAMirrorProblem(t *testing.T) {
+	was := syncDatabaseAge
+	defer func() { syncDatabaseAge = was }()
+	syncDatabaseAge = func() string { return "from 2026-10-02, less than a day old" }
+	outage := `error: failed retrieving file 'core.db' from mirror.hetzner.com : Could not resolve host: mirror.hetzner.com
+error: failed retrieving file 'core.db' from mirror.rackspace.com : Connection timed out after 10001 milliseconds
+error: failed to synchronize all databases (failed to retrieve some files)`
+	if classifyInstallFailure(outage) != installMirrors {
+		t.Fatal("no mirror answering, no 404, no signature fault: the mirrors, not the machine")
+	}
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", outage), "nodejs")
+	if err == nil || !strings.Contains(err.Error(), "mirrors or the network") || !strings.Contains(err.Error(), "less than a day old") {
+		t.Errorf("a mirror outage is said as one, with the database's age beside it:\n%v", err)
+	}
+}
+
+func TestAFailureThatIsNeitherKeepsPacmansWords(t *testing.T) {
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", "error: target not found: nodejsx"), "nodejsx")
+	if err == nil || !strings.Contains(err.Error(), "target not found") || strings.Contains(err.Error(), "package database on this machine") {
+		t.Errorf("an unknown package is pacman's own error, not a stale database:\n%v", err)
+	}
+}
+
+func TestDescribeAge(t *testing.T) {
+	now := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
+	for when, want := range map[time.Time]string{
+		now.Add(-2 * time.Hour):       "less than a day old",
+		now.Add(-5 * 24 * time.Hour):  "5 days old",
+		now.Add(-71 * 24 * time.Hour): "10 weeks old",
+	} {
+		if got := describeAge(when, now); !strings.HasSuffix(got, want) {
+			t.Errorf("%s: got %q, want suffix %q", when, got, want)
+		}
+	}
+}

internal/system/system.go

@@ -51,6 +51,9 @@ type System interface {
 
 	PackageInstalled(ctx context.Context, run Runner, name string) (bool, error)
 	InstallPackage(ctx context.Context, run Runner, name string) error
+	// RemovePackage uninstalls one package, leaving its dependencies and anything the operator
+	// changed in its configuration where the package manager leaves them (novox/hq ADR 0180).
+	RemovePackage(ctx context.Context, run Runner, name string) error
 
 	// ServiceState is "running" or "stopped". A unit that does not exist is an error, never
 	// "stopped" — reporting absence as satisfaction is the fault this host exists to prevent.