A joining machine dials the bus once the hub has answered its tunnel

Issuing the token sends the hub its new peer, and the hub applies it on
its own time; a bus dialled before then timed out naming the bus. The
first tunnel now waits for a handshake with the hub, and says so in the
tunnel's words when there is none (novox/hq ADR 0169).

cmd/mesh-host/join_tunnel.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/identity"
+)
+
+// Joining through the tunnel (novox/hq ADR 0169).
+//
+// **The bus is never open to the internet, so a joining machine reaches it over the tunnel.** It
+// makes its tunnel key first and prints the public half; the token is issued for that key, and the
+// hub is told the key before the token is shown; the token carries the one peer this machine needs.
+// So the tunnel can come up before the mesh has said anything else — the circle ADR 0004 broke by
+// carrying the bus's address in the token is broken here by carrying the hub's.
+
+// tunnelConfigPath and tunnelUnit are where the mesh's own declaration puts the private network, so
+// the first tunnel is the same interface and unit the mesh takes over, not a second one beside it.
+var (
+	tunnelConfigPath = "/etc/wireguard/mesh0.conf"
+	tunnelUnit       = "wg-quick@mesh0"
+	// lookPath finds WireGuard's tools; a variable so a test needs none installed.
+	lookPath = exec.LookPath
+)
+
+// keyCommand makes this machine's tunnel key, or reads the one it already made, and prints the
+// public half: what the token is issued for. Making it twice would be a token issued for a key the
+// machine no longer has, so an existing key is kept.
+func keyCommand(opts options) error {
+	path := identity.OverlayKeyPath(opts.state)
+	if key, err := identity.LoadOverlayKey(path); err == nil {
+		fmt.Println(key.Public)
+		return nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if _, err := os.Stat(identity.Path(opts.state)); err == nil {
+		return fmt.Errorf("this machine has joined already (%s), and its tunnel key is its own; "+
+			"there is no key to make", identity.Path(opts.state))
+	}
+	key, err := identity.GenerateOverlayKey()
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	if err := os.WriteFile(path, []byte(key.Private+"\n"), 0o600); err != nil {
+		return fmt.Errorf("cannot write this machine's tunnel key: %w", err)
+	}
+	fmt.Println(key.Public)
+	fmt.Fprintln(os.Stderr, "\nthis machine's tunnel key, made here; the private half stays in "+path+".\n"+
+		"Issue the token for it — `token issue --new <name> --overlay-key <the line above>` — and enrol with that token.")
+	return nil
+}
+
+// tunnelKeyFor is the key a token through the tunnel was issued for, read from where `key` left it.
+// Refused when there is none, or it is another: the hub knows only the key the token names.
+func tunnelKeyFor(t *identity.TokenTunnel, state string) (identity.OverlayKey, error) {
+	path := identity.OverlayKeyPath(state)
+	key, err := identity.LoadOverlayKey(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return identity.OverlayKey{}, fmt.Errorf("this token was issued for a tunnel key, and this " +
+			"machine has none: run `nox-mesh-host key` here first and issue the token for the key it prints")
+	}
+	if err != nil {
+		return identity.OverlayKey{}, err
+	}
+	if key.Public != t.Key {
+		return identity.OverlayKey{}, fmt.Errorf("this token was issued for the tunnel key %s, and this "+
+			"machine's is %s — it is another machine's token, or the key was made again; issue a new "+
+			"token for %s", t.Key, key.Public, key.Public)
+	}
+	return key, nil
+}
+
+// tunnelConfig is the first tunnel: this machine's address, and the hub as its one peer, reaching the
+// whole private network through it. The private key is set from its file, as the mesh's own
+// declaration does it, so the file holds no secret.
+func tunnelConfig(t *identity.TokenTunnel, keyPath string) string {
+	return fmt.Sprintf(`# Written by nox-mesh-host enrol: the one peer a joining machine needs (novox/hq ADR 0169).
+# The mesh's own declaration replaces this once the machine has joined.
+[Interface]
+Address = %s
+PostUp = wg set %%i private-key %s
+
+[Peer]
+PublicKey = %s
+Endpoint = %s
+AllowedIPs = %s
+PersistentKeepalive = 25
+`, t.Address, keyPath, t.HubKey, t.HubEndpoint, t.Range)
+}
+
+// bringTheTunnelUp writes the first tunnel and starts it, so the bus the token names can be reached.
+func bringTheTunnelUp(ctx context.Context, t *identity.TokenTunnel, keyPath string, run apply.Runner) error {
+	if _, err := lookPath("wg-quick"); err != nil {
+		return errors.New("joining through the tunnel needs WireGuard's tools on this machine " +
+			"(wireguard-tools), and wg-quick is not here")
+	}
+	if err := os.MkdirAll(filepath.Dir(tunnelConfigPath), 0o700); err != nil {
+		return err
+	}
+	if err := os.WriteFile(tunnelConfigPath, []byte(tunnelConfig(t, keyPath)), 0o600); err != nil {
+		return fmt.Errorf("cannot write the first tunnel: %w", err)
+	}
+	if out, err := run(ctx, "systemctl", "restart", tunnelUnit); err != nil {
+		return fmt.Errorf("the first tunnel would not start (%s): %v %s", tunnelUnit, err, strings.TrimSpace(out))
+	}
+	fmt.Printf("the tunnel to the hub is up: %s, through %s\n", t.Address, t.HubEndpoint)
+	return waitForTheHub(ctx, run, handshakeWithin)
+}
+
+// handshakeWithin is how long the hub has to answer the first tunnel. Issuing the token sent the hub
+// this machine as a peer; the hub applies that on its own time, and a bus dialled before it has is a
+// timeout that names the bus rather than the tunnel.
+var handshakeWithin = 90 * time.Second
+
+// waitForTheHub waits until the tunnel has shaken hands with the hub, so the bus is dialled over a
+// tunnel that answers — and says so in the tunnel's own words when it does not.
+func waitForTheHub(ctx context.Context, run apply.Runner, within time.Duration) error {
+	deadline := time.Now().Add(within)
+	for {
+		out, err := run(ctx, "wg", "show", "mesh0", "latest-handshakes")
+		if err == nil {
+			for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+				fields := strings.Fields(line)
+				if len(fields) == 2 && fields[1] != "0" {
+					fmt.Println("the hub answered the tunnel")
+					return nil
+				}
+			}
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("the hub has not answered the tunnel in %s: the token may be another machine's, "+
+				"the hub may not have been sent this machine as a peer, or its tunnel's port is not reachable "+
+				"from here", within)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+}

cmd/mesh-host/join_tunnel_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/identity"
+)
+
+// `key` makes the tunnel key once and prints its public half; asked again it prints the same one,
+// because a token may already have been issued for it (novox/hq ADR 0169).
+func TestKeyMakesTheTunnelKeyOnceAndKeepsIt(t *testing.T) {
+	dir := t.TempDir()
+	opts := options{state: filepath.Join(dir, "state.json")}
+	first := captureStdout(t, func() {
+		if err := keyCommand(opts); err != nil {
+			t.Fatal(err)
+		}
+	})
+	second := captureStdout(t, func() {
+		if err := keyCommand(opts); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if strings.TrimSpace(first) == "" || strings.TrimSpace(first) != strings.TrimSpace(second) {
+		t.Fatalf("the key changed between two asks: %q then %q", first, second)
+	}
+	info, err := os.Stat(identity.OverlayKeyPath(opts.state))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Mode().Perm() != 0o600 {
+		t.Errorf("the private half is readable beyond root: %v", info.Mode().Perm())
+	}
+}
+
+// A token through the tunnel takes the key it was issued for, and says so when this machine has none
+// or another.
+func TestATokenThroughTheTunnelTakesItsOwnKey(t *testing.T) {
+	dir := t.TempDir()
+	state := filepath.Join(dir, "state.json")
+	tt := &identity.TokenTunnel{Key: "x", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "h", HubEndpoint: "198.51.100.1:51820"}
+	if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "nox-mesh-host key") {
+		t.Fatalf("a machine with no key was not told to make one: %v", err)
+	}
+	captureStdout(t, func() { _ = keyCommand(options{state: state}) })
+	if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "issued for the tunnel key x") {
+		t.Fatalf("another machine's token was taken: %v", err)
+	}
+	mine, _ := identity.LoadOverlayKey(identity.OverlayKeyPath(state))
+	tt.Key = mine.Public
+	if got, err := tunnelKeyFor(tt, state); err != nil || got.Public != mine.Public {
+		t.Fatalf("this machine's own token was refused: %v", err)
+	}
+}
+
+// The first tunnel is the mesh's interface and unit, with the hub as its one peer and no secret in
+// the file — the same shape the mesh's declaration replaces it with.
+func TestTheFirstTunnelIsTheMeshsInterfaceWithTheHubAsItsPeer(t *testing.T) {
+	dir := t.TempDir()
+	tunnelConfigPath = filepath.Join(dir, "wireguard", "mesh0.conf")
+	lookPath = func(string) (string, error) { return "/usr/bin/wg-quick", nil }
+	t.Cleanup(func() { tunnelConfigPath = "/etc/wireguard/mesh0.conf" })
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "wg" {
+			return "HUBKEY\t1759400000\n", nil
+		}
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		return "", nil
+	}
+	tt := &identity.TokenTunnel{Key: "k", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "HUBKEY", HubEndpoint: "198.51.100.1:51820"}
+	captureStdout(t, func() {
+		if err := bringTheTunnelUp(context.Background(), tt, "/var/lib/mesh-host/overlay.key", run); err != nil {
+			t.Fatal(err)
+		}
+	})
+	raw, err := os.ReadFile(tunnelConfigPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := string(raw)
+	for _, want := range []string{"Address = 10.42.0.9/32", "PostUp = wg set %i private-key /var/lib/mesh-host/overlay.key",
+		"PublicKey = HUBKEY", "Endpoint = 198.51.100.1:51820", "AllowedIPs = 10.42.0.0/16", "PersistentKeepalive = 25"} {
+		if !strings.Contains(conf, want) {
+			t.Errorf("the first tunnel lacks %q:\n%s", want, conf)
+		}
+	}
+	if strings.Contains(conf, "PrivateKey") {
+		t.Error("the first tunnel's file holds the private key")
+	}
+	if len(ran) != 1 || ran[0] != "systemctl restart wg-quick@mesh0" {
+		t.Errorf("the tunnel was started as %v", ran)
+	}
+}
+
+// captureStdout is what fn printed to standard output.
+func captureStdout(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	was := os.Stdout
+	os.Stdout = w
+	fn()
+	os.Stdout = was
+	w.Close()
+	out, _ := io.ReadAll(r)
+	return string(out)
+}
+
+// The bus is dialled only once the hub has answered the tunnel, and a hub that never does is said
+// as the tunnel's fault rather than the bus's.
+func TestTheBusWaitsForTheHubToAnswer(t *testing.T) {
+	asked := 0
+	answersOnThird := func(_ context.Context, name string, args ...string) (string, error) {
+		asked++
+		if asked < 3 {
+			return "HUBKEY\t0\n", nil
+		}
+		return "HUBKEY\t1759400000\n", nil
+	}
+	captureStdout(t, func() {
+		if err := waitForTheHub(context.Background(), answersOnThird, time.Minute); err != nil {
+			t.Fatal(err)
+		}
+	})
+	never := func(context.Context, string, ...string) (string, error) { return "HUBKEY\t0\n", nil }
+	err := waitForTheHub(context.Background(), never, 0)
+	if err == nil || !strings.Contains(err.Error(), "has not answered the tunnel") {
+		t.Fatalf("a hub that never answered was not said: %v", err)
+	}
+}

cmd/mesh-host/main.go

@@ -96,6 +96,9 @@ const usage = `mesh-host — the node host
   reconcile   make this machine match what the mesh last told it — or, before any
               mesh has, the bundle this host carries
   bundle      show what this host carries
+  key         make this machine's tunnel key, or read the one it made, and print the public
+              half: what its join token is issued for (novox/hq ADR 0169)
+  enrol --token T  join the mesh — through the tunnel when the token was issued for a key
   overlay take  take over the tunnel found here (novox/hq ADR 0105): its key becomes this
               node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
   owned       what this host has applied and still owns
@@ -212,6 +215,9 @@ func parseArgs(args []string) (string, options, error) {
 func run(ctx context.Context, command string, opts options) error {
 	jsonOut, timeout := opts.json, opts.timeout
 	switch command {
+	case "key":
+		return keyCommand(opts)
+
 	case "profile":
 		p := profile.Detect(ctx, profile.Default(nil), timeout)
 		if jsonOut {
@@ -788,7 +794,18 @@ func enrol(ctx context.Context, opts options) error {
 			fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public)
 		}
 	}
-	if found == nil {
+	switch {
+	case found == nil && token.Tunnel != nil:
+		// Through the tunnel (novox/hq ADR 0169): the key `key` made, which the token names, and the
+		// tunnel brought up from the token before the bus is dialled — the bus is reached over it.
+		mine.Overlay, err = tunnelKeyFor(token.Tunnel, opts.state)
+		if err != nil {
+			return err
+		}
+		if err := bringTheTunnelUp(ctx, token.Tunnel, identity.OverlayKeyPath(opts.state), apply.ExecRunner); err != nil {
+			return err
+		}
+	case found == nil:
 		mine.Overlay, err = identity.GenerateOverlayKey()
 		if err != nil {
 			return err

examples/foundation-first-node-nats.lock

@@ -161,7 +161,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.mesh-controller.tool.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

internal/apply/apply.go

@@ -247,20 +247,6 @@ func ApplyKeeping(
 	// orphan is removed, and if a removal then fails the guard is already up. A stale opening on an
 	// adopted node is removed as any orphan is.
 	var protecting, orphans []store.Applied
-	// **What a declared process replaces is handed over, not removed first** (novox/hq issue 213).
-	// Every other orphan goes before anything is applied; one a process names under `replaces` is
-	// kept until that process is applied and still running a moment later, so whatever it was —
-	// the controller's container — answers until its replacement does, and goes on answering if
-	// the replacement never comes up.
-	replacedBy := map[string]string{}
-	for _, r := range d.Resources {
-		if p, ok := r.(*declaration.Process); ok {
-			for _, id := range p.Replaces {
-				replacedBy[id] = p.ID
-			}
-		}
-	}
-	handover := map[string][]store.Applied{}
 	for _, orphan := range known.Orphans(declared, origin) {
 		if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
 			protecting = append(protecting, orphan)
@@ -278,10 +264,6 @@ func ApplyKeeping(
 			log(fmt.Sprintf("  kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
 			continue
 		}
-		if by, replaced := replacedBy[orphan.ID]; replaced {
-			handover[by] = append(handover[by], orphan)
-			continue
-		}
 		orphans = append(orphans, orphan)
 	}
 	ordered := d.Resources
@@ -535,11 +517,6 @@ func ApplyKeeping(
 			if c, ok := resource.(*declaration.Container); ok && c.RunOnce {
 				gates = true
 			}
-			// And a run-once process, which is the same step hosted as a unit: a version whose
-			// preparation did not complete must not be started (novox/hq ADR 0135, issue 213).
-			if p, ok := resource.(*declaration.Process); ok && p.RunOnce {
-				gates = true
-			}
 			if gates {
 				failed.Gated = true
 				// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
@@ -630,28 +607,6 @@ func ApplyKeeping(
 			}
 			log(line)
 		}
-
-		// Its replacement applied: what it replaces goes now, once it is seen running (issue 213).
-		if waiting, has := handover[resource.Identity()]; has {
-			delete(handover, resource.Identity())
-			if err := handOver(ctx, run, resource, waiting, removeOrphan, &report, log); err != nil {
-				failures = append(failures, err)
-			}
-		}
-	}
-
-	// A replacement that did not apply — failed, skipped behind its module's step, held — leaves
-	// what it replaces running and recorded, said, for the next apply to hand over.
-	unhanded := make([]string, 0, len(handover))
-	for by := range handover {
-		unhanded = append(unhanded, by)
-	}
-	sort.Strings(unhanded)
-	for _, by := range unhanded {
-		for _, orphan := range handover[by] {
-			keptForReplacement(&report, log, orphan,
-				fmt.Sprintf("kept: %s, which replaces it, did not apply", by))
-		}
 	}
 
 	if !orphansRemoved {
@@ -1086,38 +1041,6 @@ type unitReloader interface {
 	ReloadUnits(ctx context.Context, run system.Runner) error
 }
 
-// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
-// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
-var serviceSettle = 2 * time.Second
-
-// stayedRunning is the state of a unit the host has just asked to run, read twice.
-//
-// **Because the first read races the failure.** A service manager returns when it has started the
-// process, and the unit is "activating" or "active" at that instant whatever the process is about
-// to do. A daemon that reads its configuration, refuses it and exits does so a fraction of a second
-// later — fail2ban took 221 milliseconds the day this was written — so a single read back says
-// running about a machine whose daemon is already gone, and the apply reports "restarted" for a
-// service that is dead. Every ban on both public machines was lost that way while every check
-// passed (novox/hq ADR 0184), which is the one shape of failure this host exists to refuse.
-//
-// So it looks again, after the moment in which that happens. It does not wait for a slow unit to
-// finish starting: a unit still coming up reads as running both times and is accepted, as before.
-// What this catches is a unit that was running and is not any more.
-func stayedRunning(ctx context.Context, sys system.System, run Runner, unit string) (string, error) {
-	state, err := sys.ServiceState(ctx, run, unit)
-	if err != nil || state != "running" {
-		return state, err
-	}
-	timer := time.NewTimer(serviceSettle)
-	defer timer.Stop()
-	select {
-	case <-ctx.Done():
-		return state, ctx.Err()
-	case <-timer.C:
-	}
-	return sys.ServiceState(ctx, run, unit)
-}
-
 func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
 	changed map[string]bool, previous store.Applied) (Outcome, error) {
 	if r.Stateless() {
@@ -1197,9 +1120,6 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 		// Read back. A service manager accepting a command says the transaction was accepted,
 		// not that the unit is running — one that starts and immediately dies satisfies it.
 		after, err := sys.ServiceState(ctx, run, r.Unit)
-		if err == nil && r.State == "running" {
-			after, err = stayedRunning(ctx, sys, run, r.Unit)
-		}
 		if err != nil {
 			return out, err
 		}
@@ -1220,7 +1140,7 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 		}
 		// Read back, for the same reason as above: a unit that starts and immediately dies
 		// satisfies a service manager and nothing else.
-		after, err := stayedRunning(ctx, sys, run, r.Unit)
+		after, err := sys.ServiceState(ctx, run, r.Unit)
 		if err != nil {
 			return out, err
 		}
@@ -1310,7 +1230,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service,
 	}
 	// Read back: it was running, and a restart or reload that left it otherwise is a failure —
 	// the machine's network manager down is not a change to report and move past.
-	after, err := stayedRunning(ctx, sys, run, r.Unit)
+	after, err := sys.ServiceState(ctx, run, r.Unit)
 	if err != nil {
 		return out, err
 	}
@@ -1488,7 +1408,7 @@ func applyPackage(ctx context.Context, sys system.System, r *declaration.Package
 		return out, err
 	}
 	if r.Absent {
-		// Declared absent (novox/hq ADR 0180): removed when it is here, left alone when it is not.
+		// Declared absent (novox/hq ADR 0175): removed when it is here, left alone when it is not.
 		if !installed {
 			out.Action = "unchanged"
 			out.Detail = "not installed, as declared"
@@ -2498,102 +2418,3 @@ func moduleOf(identity string) (string, bool) {
 	}
 	return identity[:at], true
 }
-
-// handOver removes what a process replaces, once the process is running and still is a moment later
-// (novox/hq issue 213, ADR 0184). A replacement that is not up keeps what it replaces in place
-// and recorded, and is this apply's failure: the old one answers until a later apply finds the new
-// one up.
-func handOver(ctx context.Context, run Runner, resource declaration.Resource,
-	waiting []store.Applied, removeOrphan func(store.Applied) error, report *Report,
-	log func(string)) *Error {
-	p, ok := resource.(*declaration.Process)
-	if !ok {
-		return nil
-	}
-	if why := stillUp(ctx, run, p.Name+".service"); why != "" {
-		for _, orphan := range waiting {
-			keptForReplacement(report, log, orphan,
-				fmt.Sprintf("kept: %s, which replaces it, is not running (%s)", p.ID, why))
-		}
-		return &Error{Resource: p.ID, Err: fmt.Errorf(
-			"%s was started and is not running a moment later (%s), so what it replaces was kept: %s",
-			p.Name, why, appliedIDs(waiting)), Done: *report}
-	}
-	for _, orphan := range waiting {
-		if err := removeOrphan(orphan); err != nil {
-			var failed *Error
-			if errors.As(err, &failed) {
-				return failed
-			}
-			return &Error{Resource: orphan.ID, Err: err, Done: *report}
-		}
-	}
-	return nil
-}
-
-// handoverSettle is how long a replacement must stay up before what it replaces goes. Longer than a
-// service's second look (serviceSettle): this one decides whether the last thing answering is
-// removed, and a process that fails on its store or its bus does so after it opened them, not in the
-// first instant. A test sets it to nothing.
-var handoverSettle = 10 * time.Second
-
-// stillUp says why a process's unit is not up and staying up, or nothing when it is: active and
-// running at two looks handoverSettle apart, the same main process both times, restarted by
-// nothing in between. Stricter than stayedRunning, which reads "activating" as running — the state
-// a crash-looping unit is in while it waits to be started again, which is exactly the replacement
-// that must not be handed anything.
-func stillUp(ctx context.Context, run Runner, unit string) string {
-	look := func() (map[string]string, string) {
-		out, err := run(ctx, "systemctl", "show", unit, "--property=ActiveState", "--property=SubState",
-			"--property=MainPID", "--property=NRestarts")
-		if err != nil {
-			return nil, err.Error()
-		}
-		got := map[string]string{}
-		for _, line := range strings.Split(out, "\n") {
-			if key, value, found := strings.Cut(strings.TrimSpace(line), "="); found {
-				got[key] = value
-			}
-		}
-		if got["ActiveState"] != "active" || got["SubState"] != "running" {
-			return got, fmt.Sprintf("%s/%s", got["ActiveState"], got["SubState"])
-		}
-		return got, ""
-	}
-	first, why := look()
-	if why != "" {
-		return why
-	}
-	timer := time.NewTimer(handoverSettle)
-	defer timer.Stop()
-	select {
-	case <-ctx.Done():
-		return ctx.Err().Error()
-	case <-timer.C:
-	}
-	second, why := look()
-	if why != "" {
-		return why
-	}
-	if first["MainPID"] != second["MainPID"] || first["NRestarts"] != second["NRestarts"] {
-		return fmt.Sprintf("restarted while it was watched (pid %s → %s, restarts %s → %s)",
-			first["MainPID"], second["MainPID"], first["NRestarts"], second["NRestarts"])
-	}
-	return ""
-}
-
-// keptForReplacement reports an orphan kept because what replaces it is not yet in its place.
-func keptForReplacement(report *Report, log func(string), orphan store.Applied, detail string) {
-	report.Outcomes = append(report.Outcomes, Outcome{
-		ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "kept", Detail: detail,
-	})
-	log(fmt.Sprintf("  kept %s (%s): %s", orphan.ID, orphan.Target, strings.TrimPrefix(detail, "kept: ")))
-}
-
-func appliedIDs(applied []store.Applied) string {
-	ids := make([]string, 0, len(applied))
-	for _, a := range applied {
-		ids = append(ids, a.ID)
-	}
-	return strings.Join(ids, ", ")
-}

internal/apply/archive.go

@@ -66,10 +66,16 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
 		}
 	}
 
-	written, err := replaceWith(body, r.Path, r.Owner)
+	if err := os.MkdirAll(r.Path, 0o755); err != nil {
+		return out, err
+	}
+	written, err := unpack(body, r.Path)
 	if err != nil {
 		return out, err
 	}
+	if err := ownAll(r.Path, r.Owner); err != nil {
+		return out, err
+	}
 	out.Action = "updated"
 	if previous.Wrote == "" {
 		out.Action = "created"
@@ -78,63 +84,6 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
 	return out, nil
 }
 
-// replaceWith makes the directory exactly the archive (novox/hq issue 220).
-//
-// **The tree on disk is the archive and nothing else.** The digest is the whole identity of what
-// is unpacked here, so a file the previous archive had and this one does not must go. Unpacked over
-// the old tree, it stayed: a bundle rebuilt as one file per entrypoint kept the package directory
-// of the version before, which code could still import, and a fix that removed a file worked on a
-// fresh machine only. So the archive is unpacked into a fresh directory beside the old one, owned,
-// and swapped in by rename. A running process keeps the files it has open, and the old tree is
-// removed only once the new one is in place. A failed unpack leaves the old tree untouched.
-func replaceWith(body []byte, path, owner string) (int, error) {
-	parent := filepath.Dir(path)
-	if err := os.MkdirAll(parent, 0o755); err != nil {
-		return 0, err
-	}
-	fresh := path + ".unpacking"
-	replaced := path + ".replaced"
-	// What an interrupted earlier attempt left beside the directory.
-	for _, leftover := range []string{fresh, replaced} {
-		if err := os.RemoveAll(leftover); err != nil {
-			return 0, err
-		}
-	}
-	if err := os.Mkdir(fresh, 0o755); err != nil {
-		return 0, err
-	}
-	written, err := unpack(body, fresh)
-	if err == nil {
-		err = ownAll(fresh, owner)
-	}
-	if err != nil {
-		os.RemoveAll(fresh)
-		return written, err
-	}
-	hadOne := true
-	if err := os.Rename(path, replaced); err != nil {
-		if !os.IsNotExist(err) {
-			os.RemoveAll(fresh)
-			return written, err
-		}
-		hadOne = false
-	}
-	if err := os.Rename(fresh, path); err != nil {
-		if hadOne {
-			// Put the old tree back rather than leave nothing at the path.
-			os.Rename(replaced, path)
-		}
-		os.RemoveAll(fresh)
-		return written, err
-	}
-	if hadOne {
-		if err := os.RemoveAll(replaced); err != nil {
-			return written, fmt.Errorf("%s is in place, and the tree it replaced could not be removed: %w", path, err)
-		}
-	}
-	return written, nil
-}
-
 func fetch(ctx context.Context, source string) ([]byte, error) {
 	request, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
 	if err != nil {

internal/apply/archive_replace_test.go

@@ -1,68 +0,0 @@
-package apply
-
-import (
-	"context"
-	"os"
-	"testing"
-
-	"github.com/novox/mesh-host/internal/store"
-)
-
-// novox/hq issue 220: the tree on disk is exactly the archive. A file the previous archive had and
-// this one does not is gone, and nothing is left beside the directory.
-func TestAnArchiveReplacesTheTreeItWasUnpackedOver(t *testing.T) {
-	dir := t.TempDir()
-	first, firstDigest := anArchive(t, map[string]string{"index.js": "old", "node_modules/dep/index.js": "dep"})
-	d := declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, first)+
-		`","digest":"`+firstDigest+`","path":"`+dir+`/tools"}`)
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	second, secondDigest := anArchive(t, map[string]string{"index.js": "one file"})
-	d = declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, second)+
-		`","digest":"`+secondDigest+`","path":"`+dir+`/tools"}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil); err != nil {
-		t.Fatal(err)
-	}
-	if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "one file" {
-		t.Fatalf("index.js is %q", got)
-	}
-	if _, err := os.Stat(dir + "/tools/node_modules"); err == nil {
-		t.Fatal("the previous archive's directory is still there")
-	}
-	entries, _ := os.ReadDir(dir)
-	if len(entries) != 1 {
-		names := []string{}
-		for _, e := range entries {
-			names = append(names, e.Name())
-		}
-		t.Fatalf("beside the tree: %v", names)
-	}
-}
-
-// A tree is replaced only by one that unpacked whole: an archive refused halfway leaves the old
-// tree as it was.
-func TestARefusedArchiveLeavesTheTreeItWouldHaveReplaced(t *testing.T) {
-	dir := t.TempDir()
-	first, firstDigest := anArchive(t, map[string]string{"index.js": "old"})
-	d := declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, first)+
-		`","digest":"`+firstDigest+`","path":"`+dir+`/tools"}`)
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	bad, badDigest := anArchive(t, map[string]string{"index.js": "new", "../../escaped": "no"})
-	d = declare(t, `{"id":"bundle-tools","type":"archive","source":"`+serving(t, bad)+
-		`","digest":"`+badDigest+`","path":"`+dir+`/tools"}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil); err == nil {
-		t.Fatal("an escaping archive was accepted")
-	}
-	if got, _ := os.ReadFile(dir + "/tools/index.js"); string(got) != "old" {
-		t.Fatalf("index.js is %q after a refused archive", got)
-	}
-	if entries, _ := os.ReadDir(dir); len(entries) != 1 {
-		t.Fatalf("%d entries beside the tree after a refused archive", len(entries))
-	}
-}

internal/apply/handover_test.go

@@ -1,318 +0,0 @@
-package apply
-
-import (
-	"context"
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/novox/mesh-host/internal/store"
-)
-
-// novox/hq issue 213: the controller moves from a container to a process on the one machine that
-// runs it. Every orphan is removed before anything is applied, so without a handover the container
-// went first and nothing answered the mesh's verbs while the process was fetched, unpacked and
-// started — and for ever, if it did not start. A process that `replaces` the container is applied
-// first; the container goes only once the process is running a moment later.
-
-// aMachine fakes the service manager and the container runtime: it records every command, answers
-// `systemctl show` with whether the process is running, and has a container until it is removed.
-type aMachine struct {
-	commands  []string
-	running   bool // what `systemctl show` says of the process once it was started
-	started   bool
-	container bool
-	timer     bool // whether a timer, once started, is up
-	crashing  bool // up at the first look, waiting to restart at the next
-	looks     int
-}
-
-func (m *aMachine) run(ctx context.Context, name string, args ...string) (string, error) {
-	line := name + " " + strings.Join(args, " ")
-	m.commands = append(m.commands, line)
-	switch {
-	case name == "systemctl" && len(args) > 0 && (args[0] == "restart" || args[0] == "start"):
-		m.started = true
-	case name == "systemctl" && len(args) > 0 && args[0] == "is-active" && strings.HasSuffix(args[len(args)-1], ".timer"):
-		if m.timer {
-			return "active", nil
-		}
-		return "inactive", errors.New("inactive")
-	case name == "systemctl" && len(args) > 0 && args[0] == "is-active":
-		if m.started && m.running {
-			return "active", nil
-		}
-		return "inactive", errors.New("inactive")
-	case name == "systemctl" && len(args) > 0 && args[0] == "show":
-		m.looks++
-		if m.crashing && m.started {
-			// Up at the first look; waiting to be started again, a new process, at the second.
-			if m.looks == 1 {
-				return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
-			}
-			return "ActiveState=activating\nSubState=auto-restart\nMainPID=0\nNRestarts=1\n", nil
-		}
-		if m.started && m.running {
-			return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
-		}
-		return "ActiveState=inactive\nSubState=dead\nMainPID=0\nNRestarts=0\n", nil
-	case name == "docker" && len(args) > 1 && args[0] == "rm":
-		m.container = false
-	case name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect":
-		if !m.container {
-			return "", errors.New("no such container")
-		}
-		return "true\t", nil
-	}
-	return "", nil
-}
-
-func (m *aMachine) index(prefix string) int {
-	for i, c := range m.commands {
-		if strings.HasPrefix(c, prefix) {
-			return i
-		}
-	}
-	return -1
-}
-
-// theController is a machine whose controller ran as a container, recorded, and a declaration that
-// runs it as a process from a bundle served here instead.
-func theController(t *testing.T, digest, source, extra string) (store.State, string) {
-	t.Helper()
-	known := store.State{Resources: []store.Applied{{
-		Origin: store.OriginDeclared, ID: "mesh-controller.server", Type: "container", Target: "mesh-controller",
-	}}}
-	return known, `{"declaration":1,"resources":[
-	  {"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + source +
-		`","digest":"` + digest + `","run":["./mesh-controller","serve"]` + extra + `}]}`
-}
-
-func onAMachine(t *testing.T) {
-	t.Helper()
-	serviceSettle, handoverSettle = 0, 0
-	wasUnits, wasBundles := unitDir, daemonRoot
-	unitDir, daemonRoot = t.TempDir(), t.TempDir()
-	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
-}
-
-func TestAContainerAProcessReplacesGoesOnlyOnceTheProcessRuns(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
-	m := &aMachine{running: true, container: true}
-
-	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
-		store.OriginDeclared, m.run, nil, nil)
-	if err != nil {
-		t.Fatalf("the handover failed: %v", err)
-	}
-	started, removed := m.index("systemctl restart mesh-controller.service"), m.index("docker rm -f mesh-controller")
-	if started < 0 || removed < 0 {
-		t.Fatalf("the process was not started or the container not removed: %v", m.commands)
-	}
-	if removed < started {
-		t.Fatalf("the container was removed before its replacement was started — a window with "+
-			"nothing answering: %v", m.commands)
-	}
-	if looked := m.index("systemctl show mesh-controller.service"); looked < 0 || looked > removed {
-		t.Errorf("the container was removed without looking whether the process runs: %v", m.commands)
-	}
-	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
-		t.Errorf("the container's outcome is %+v, want removed", o)
-	}
-	if _, still := after.Find("mesh-controller.server"); still {
-		t.Error("the host still records the container it removed")
-	}
-	if _, has := after.Find("mesh-controller.controller"); !has {
-		t.Error("the process was not recorded")
-	}
-}
-
-// The case the handover exists for: the replacement does not stay up. The container keeps
-// answering, stays recorded so a later apply hands it over, and the apply says why it failed.
-func TestAContainerIsKeptWhenItsReplacementDoesNotRun(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
-	m := &aMachine{running: false, container: true}
-
-	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
-		store.OriginDeclared, m.run, nil, nil)
-	if err == nil {
-		t.Fatal("a replacement that is not running was reported as a clean apply")
-	}
-	if !strings.Contains(err.Error(), "mesh-controller.server") {
-		t.Errorf("the failure does not name what was kept: %v", err)
-	}
-	if m.index("docker rm") >= 0 {
-		t.Fatalf("the container was removed though its replacement is not running: %v", m.commands)
-	}
-	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
-		t.Errorf("the container's outcome is %+v, want kept", o)
-	}
-	if _, still := after.Find("mesh-controller.server"); !still {
-		t.Fatal("the container was forgotten, so no later apply would ever remove it")
-	}
-
-	// The next apply finds the process up and finishes the handover.
-	m.running, m.commands = true, nil
-	report, after, err = Apply(context.Background(), archHost(t), parse(t, raw), after,
-		store.OriginDeclared, m.run, nil, nil)
-	if err != nil {
-		t.Fatalf("the second apply failed: %v", err)
-	}
-	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
-		t.Errorf("the second apply did not hand over: %+v (%v)", o, m.commands)
-	}
-	if _, still := after.Find("mesh-controller.server"); still {
-		t.Error("the container is still recorded after the handover")
-	}
-}
-
-// A replacement that never applied — its bundle is not what was declared — touches nothing, and
-// what it replaces keeps running.
-func TestAContainerIsKeptWhenItsReplacementFailsToApply(t *testing.T) {
-	onAMachine(t)
-	body, _ := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	known, raw := theController(t, "sha256:"+strings.Repeat("b", 64), serving(t, body),
-		`,"replaces":["mesh-controller.server"]`)
-	m := &aMachine{running: true, container: true}
-
-	report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
-		store.OriginDeclared, m.run, nil, nil)
-	if err == nil {
-		t.Fatal("a replacement whose bundle did not match was reported applied")
-	}
-	if m.index("docker rm") >= 0 {
-		t.Fatalf("the container was removed though nothing replaced it: %v", m.commands)
-	}
-	if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
-		t.Errorf("the container's outcome is %+v, want kept", o)
-	}
-	if _, still := after.Find("mesh-controller.server"); !still {
-		t.Fatal("the container was forgotten")
-	}
-}
-
-// Without `replaces` nothing changes: an orphan goes before anything is applied, as it always has.
-// Kept as a test because it is the window the field exists to close.
-func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	known, raw := theController(t, digest, serving(t, body), ``)
-	m := &aMachine{running: true, container: true}
-	if _, _, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
-		store.OriginDeclared, m.run, nil, nil); err != nil {
-		t.Fatal(err)
-	}
-	if removed, started := m.index("docker rm -f mesh-controller"), m.index("systemctl restart mesh-controller.service"); removed < 0 || removed > started {
-		t.Fatalf("an orphan nothing replaces was not removed first: %v", m.commands)
-	}
-}
-
-// novox/hq issue 213, beyond what the oneshot unit (process_step_test.go) already holds: a step
-// written `./name` runs its own bundle's binary — the controller's preparation is its own binary —
-// and a step is started, never enabled.
-func TestAStepRunsItsOwnBundlesBinaryAndIsNotEnabled(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	m := &aMachine{running: true}
-	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
-	  "source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil); err != nil {
-		t.Fatalf("the step failed: %v", err)
-	}
-	for _, c := range m.commands {
-		if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") {
-			t.Errorf("the step was run directly or enabled: %v", m.commands)
-		}
-	}
-	unit, err := os.ReadFile(filepath.Join(unitDir, "mesh-controller-prepare.service"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	want := "ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare"
-	if !strings.Contains(string(unit), want) {
-		t.Errorf("the step does not run its own bundle's binary (%q):\n%s", want, unit)
-	}
-}
-
-func TestAStepThatFailsGatesItsModule(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	src := serving(t, body)
-	run := func(ctx context.Context, name string, args ...string) (string, error) {
-		if name == "systemctl" && len(args) > 1 && args[0] == "start" {
-			return "", errors.New("Job for mesh-controller-prepare.service failed")
-		}
-		if name == "systemctl" && len(args) > 0 && args[0] == "restart" {
-			t.Errorf("the module's process was started after its step failed")
-		}
-		return "", nil
-	}
-	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
-	  "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true},
-	  {"id":"mesh-controller.controller","type":"process","name":"mesh-controller",
-	  "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","serve"]}`)
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
-	if err == nil {
-		t.Fatal("a failed step was reported as a clean apply")
-	}
-	if o := outcomeOf(report, "mesh-controller.controller"); o.Action != "skipped" {
-		t.Errorf("the process after a failed step was %+v, want skipped", o)
-	}
-}
-
-// A completed step is done: applied again unchanged, it is not run again — its service is never up
-// between runs, and reading that as "a daemon that stopped" re-ran the controller's preparation on
-// every apply. Nor is a scheduled run started off its cadence; its timer is what is kept up.
-func TestACompletedStepIsNotRunAgainAndAScheduleIsItsTimer(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"job": "#!/bin/sh\n"})
-	src := serving(t, body)
-	for _, mode := range []string{`"run-once":true`, `"schedule":"0 3 * * *"`} {
-		// The service of either is never up between runs; a scheduled one's timer is.
-		m := &aMachine{running: false, timer: true}
-		d := declare(t, `{"id":"m.job","type":"process","name":"m-job","source":"`+src+`","digest":"`+digest+
-			`","run":["./job"],`+mode+`}`)
-		_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
-		if err != nil {
-			t.Fatalf("%s: first apply: %v", mode, err)
-		}
-		m.commands = nil
-		report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, nil)
-		if err != nil {
-			t.Fatalf("%s: second apply: %v", mode, err)
-		}
-		if m.index("systemctl start m-job.service") >= 0 || m.index("systemctl restart m-job.service") >= 0 {
-			t.Errorf("%s: an unchanged apply ran the job again: %v", mode, m.commands)
-		}
-		if o := outcomeOf(report, "m.job"); o.Action != "unchanged" {
-			t.Errorf("%s: an unchanged apply reported %+v", mode, o)
-		}
-	}
-}
-
-// A replacement crash-looping between its restarts is not up, though the service manager calls it
-// "activating" — the state the host's ordinary second look accepts as running. Removing the
-// container on that reading would leave nothing answering.
-func TestAContainerIsKeptWhenItsReplacementIsCrashLooping(t *testing.T) {
-	onAMachine(t)
-	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
-	known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
-	m := &aMachine{crashing: true, container: true}
-	_, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
-		store.OriginDeclared, m.run, nil, nil)
-	if err == nil || !strings.Contains(err.Error(), "auto-restart") {
-		t.Fatalf("a crash-looping replacement was accepted: %v", err)
-	}
-	if m.index("docker rm") >= 0 {
-		t.Fatalf("the container was removed for a replacement that keeps dying: %v", m.commands)
-	}
-	if _, still := after.Find("mesh-controller.server"); !still {
-		t.Fatal("the container was forgotten")
-	}
-}

internal/apply/left_out_test.go

@@ -174,7 +174,7 @@ func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
 	}
 }
 
-// A package may be declared absent (novox/hq ADR 0180): removed when it is installed, read back,
+// A package may be declared absent (novox/hq ADR 0175): removed when it is installed, read back,
 // left alone when it is not.
 func TestAPackageDeclaredAbsentIsRemovedWhenPresentAndLeftWhenNot(t *testing.T) {
 	installed := true

internal/apply/opening.go

@@ -77,7 +77,7 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri
 		return "", nil
 	}
 	if !firewall.Installed(ctx, run) {
-		// Uninstalled (novox/hq ADR 0180): retired for good, by the module that replaced it. Said
+		// Uninstalled (novox/hq ADR 0175): retired for good, by the module that replaced it. Said
 		// once, and nothing is asked of a command that is not there.
 		if rec.RetiredBy != firewall.RetiredRemoved {
 			rec.RetiredBy = firewall.RetiredRemoved

internal/apply/opening_test.go

@@ -525,7 +525,7 @@ func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
 }
 
 // A front end that is no longer installed is recorded as removed, said once, and asked nothing of
-// (novox/hq ADR 0180).
+// (novox/hq ADR 0175).
 func TestAnUninstalledFrontEndIsRetiredForGood(t *testing.T) {
 	dir := t.TempDir()
 	u := &ufwMachine{installed: false, ruleset: "table inet mesh\n"}

internal/apply/process.go

@@ -77,26 +77,11 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 		// Everything about it is as declared. Still asked whether it is RUNNING, because a
 		// declaration that is satisfied by a record rather than by the machine is how a stopped
 		// service reports success.
-		// **The record is carried forward, not re-derived.** An unchanged outcome is recorded
-		// like any other, so one that said nothing about what was written erased the digest; the
-		// next apply then found no record, re-created the daemon, and the one after that found a
-		// record again — the node's runtime restarted every other cycle (novox/hq 04-ISSUES/210).
-		out.wrote = want
-		// **A step that ran is done, and a schedule is its timer** (novox/hq issue 213). Neither's
-		// service is meant to be up between runs, so asking whether it is — and starting it when it
-		// was not — ran a completed step again on every apply, and a scheduled run off its cadence.
-		if r.RunOnce {
-			return out, nil
-		}
-		unit := r.Name + ".service"
-		if r.Schedule != "" {
-			unit = r.Name + ".timer"
-		}
-		if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
+		if active, err := run(ctx, "systemctl", "is-active", "--quiet", r.Name+".service"); err == nil {
 			_ = active
 			return out, nil
 		}
-		if _, err := run(ctx, "systemctl", "start", unit); err != nil {
+		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
 			return out, fmt.Errorf("%s is installed and would not start: %w", r.Name, err)
 		}
 		out.Action = "updated"
@@ -125,22 +110,9 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 	// so the machine is not asked to start something that needed a migration that did not happen.
 	// Nothing is left behind to ask afterwards: the record that it ran is the digest, which is why
 	// the identity above includes the command.
-	//
-	// **Run as the unit a daemon would be, once** (novox/hq design 38 WP4c). Run directly, the
-	// step started in the host's own working directory, without its environment, its environment
-	// files or its user — `node bootstrap/index.js` resolved from wherever the host ran and was told
-	// none of the words it was declared with. A oneshot unit carries all four exactly as a daemon's
-	// does, and starting one waits for it to finish and fails when it fails.
 	if r.RunOnce {
-		unit := filepath.Join(unitDir, r.Name+".service")
-		if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
-			return out, err
-		}
-		if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
-			return out, err
-		}
-		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
-			return out, fmt.Errorf("the %s step did not complete (journalctl -u %s.service says why): %w", r.Name, r.Name, err)
+		if _, err := run(ctx, r.Run[0], r.Run[1:]...); err != nil {
+			return out, fmt.Errorf("the %s step did not complete: %w", r.Name, err)
 		}
 		out.Action = "created"
 		if previous.Wrote != "" {
@@ -237,9 +209,9 @@ func unitFor(r *declaration.Process) string {
 	if r.User != "" {
 		fmt.Fprintf(&b, "User=%s\n", r.User)
 	}
-	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
-	if r.Schedule != "" || r.RunOnce {
-		// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
+	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(r.Run, " "))
+	if r.Schedule != "" {
+		// Started by its timer and expected to finish. Restarting it would have it run
 		// continuously between fires, which is the opposite of a schedule.
 		b.WriteString("Type=oneshot\n")
 		b.WriteString("\n")
@@ -369,15 +341,3 @@ func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, st
 	}
 	return "removed", "stopped; its unit and its bundle removed — the mesh's own code", nil
 }
-
-// runFrom is the command as the unit runs it. **A command written `./name` is that file in the
-// process's own unpacked bundle** (novox/hq ADR 0193): a bundle compiled to a binary runs itself,
-// and only the host knows where it unpacked it, while the service manager takes an absolute path or
-// a name it finds on its own search path — never one relative to the working directory.
-func runFrom(r *declaration.Process) []string {
-	run := append([]string(nil), r.Run...)
-	if len(run) > 0 && strings.HasPrefix(run[0], "./") {
-		run[0] = filepath.Join(daemonRoot, r.Name, strings.TrimPrefix(run[0], "./"))
-	}
-	return run
-}

internal/apply/process_step_test.go

@@ -1,81 +0,0 @@
-package apply
-
-import (
-	"context"
-	"errors"
-	"os"
-	"os/user"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/novox/mesh-host/internal/store"
-)
-
-// A run-once process is a step run where, how and as whom it was declared (novox/hq design 38
-// WP4c): its bundle's directory, its environment and environment files, its user. Run directly,
-// it started in the host's own directory with none of them.
-func TestARunOnceProcessRunsAsItsOneshotUnit(t *testing.T) {
-	units, bundles := t.TempDir(), t.TempDir()
-	wasUnits, wasBundles := unitDir, daemonRoot
-	unitDir, daemonRoot = units, bundles
-	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
-
-	me, err := user.Current()
-	if err != nil {
-		t.Fatal(err)
-	}
-	body, digest := anArchive(t, map[string]string{"bootstrap/index.js": "console.log(1)\n"})
-	var commands []string
-	fail := false
-	run := func(ctx context.Context, name string, args ...string) (string, error) {
-		commands = append(commands, name+" "+strings.Join(args, " "))
-		if fail && name == "systemctl" && len(args) > 0 && args[0] == "start" {
-			return "", errors.New("exit status 1")
-		}
-		return "", nil
-	}
-	d := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
-		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"user":"`+me.Username+`",`+
-		`"env":{"MESH_ADMIN":"mesh-admin"},"env-file":["/var/lib/mesh/mosquitto/bootstrap.env"]}`)
-
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil); err != nil {
-		t.Fatal(err)
-	}
-	unit, err := os.ReadFile(filepath.Join(units, "mosquitto-bootstrap.service"))
-	if err != nil {
-		t.Fatalf("no unit was written for the step: %v", err)
-	}
-	for _, want := range []string{
-		"WorkingDirectory=" + filepath.Join(bundles, "mosquitto-bootstrap"),
-		"EnvironmentFile=/var/lib/mesh/mosquitto/bootstrap.env",
-		`Environment="MESH_ADMIN=mesh-admin"`,
-		"User=" + me.Username,
-		"Type=oneshot",
-		"ExecStart=node bootstrap/index.js",
-	} {
-		if !strings.Contains(string(unit), want) {
-			t.Errorf("the step's unit lacks %q:\n%s", want, unit)
-		}
-	}
-	for _, never := range []string{"Restart=always", "[Install]", "Type=simple"} {
-		if strings.Contains(string(unit), never) {
-			t.Errorf("a step's unit says %q:\n%s", never, unit)
-		}
-	}
-	joined := strings.Join(commands, "; ")
-	if !strings.Contains(joined, "systemctl start mosquitto-bootstrap.service") {
-		t.Errorf("the step was not started as its unit: %s", joined)
-	}
-	if strings.Contains(joined, "node bootstrap/index.js") || strings.Contains(joined, "enable mosquitto-bootstrap") {
-		t.Errorf("the step was run directly or enabled: %s", joined)
-	}
-
-	// A step that fails fails the apply, and is not recorded as done.
-	fail = true
-	d2 := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
-		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"env":{"MESH_ADMIN":"changed"}}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, run, nil, nil); err == nil {
-		t.Error("a step that failed did not fail the apply")
-	}
-}

internal/apply/process_test.go

@@ -256,73 +256,3 @@ func TestAProcessRecordedUnderAPathlikeNameIsRefusedNotRemoved(t *testing.T) {
 		}
 	}
 }
-
-// novox/hq 04-ISSUES/210: the node's runtime was re-created — and restarted — on every reconcile,
-// because the host did not find what it wrote for a process the cycle before. Applying the same
-// process declaration twice must do no work the second time.
-func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
-	units, bundles := t.TempDir(), t.TempDir()
-	wasUnits, wasBundles := unitDir, daemonRoot
-	unitDir, daemonRoot = units, bundles
-	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
-
-	body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"})
-	var commands []string
-	run := func(ctx context.Context, name string, args ...string) (string, error) {
-		commands = append(commands, name+" "+strings.Join(args, " "))
-		return "", nil
-	}
-	d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
-		`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
-
-	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
-		t.Fatalf("first apply: %+v, want created", o)
-	}
-	rec, ok := state.Find("node-tools.runtime")
-	if !ok || rec.Wrote == "" {
-		t.Fatalf("the host did not record what it wrote for the process: %+v", rec)
-	}
-	commands = nil
-	again, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o := outcomeOf(again, "node-tools.runtime"); o.Action != "unchanged" {
-		t.Errorf("second apply: %+v, want unchanged", o)
-	}
-	for _, c := range commands {
-		if strings.Contains(c, "restart") {
-			t.Errorf("the second apply restarted the process: %v", commands)
-		}
-	}
-	// And the record survives an unchanged apply: the third cycle is unchanged too. This is the
-	// cycle the live mesh showed — created, unchanged, created — before the record was carried.
-	if rec, _ := state.Find("node-tools.runtime"); rec.Wrote == "" {
-		t.Fatalf("an unchanged apply dropped the digest from the record: %+v", rec)
-	}
-	commands = nil
-	third, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, run, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o := outcomeOf(third, "node-tools.runtime"); o.Action != "unchanged" {
-		t.Errorf("third apply: %+v, want unchanged", o)
-	}
-}
-
-// novox/hq ADR 0193: a bundle compiled to a binary runs itself — `./name` is that file in the
-// process's own unpacked bundle, made absolute because the service manager takes nothing relative.
-func TestAProcessRunsItsOwnBundlesBinary(t *testing.T) {
-	unit := unitFor(&declaration.Process{Name: "node-tools", Run: []string{"./node-tools", "serve"}})
-	if !strings.Contains(unit, "ExecStart="+filepath.Join(daemonRoot, "node-tools", "node-tools")+" serve\n") {
-		t.Errorf("the binary is not run from its bundle:\n%s", unit)
-	}
-	other := unitFor(&declaration.Process{Name: "x", Run: []string{"node", "src/main.js"}})
-	if !strings.Contains(other, "ExecStart=node src/main.js\n") {
-		t.Errorf("a command found on the path was changed:\n%s", other)
-	}
-}

internal/apply/stayed_running_test.go

@@ -1,100 +0,0 @@
-package apply
-
-import (
-	"context"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/novox/mesh-host/internal/store"
-)
-
-// A daemon that reads a configuration it refuses dies a fraction of a second after the service
-// manager has reported it started — fail2ban took 221 milliseconds on the control node the day this
-// was written. One read back catches nothing: the unit is active at that instant. The mesh reported
-// the service "restarted" while every ban on two public machines was gone, and every check passed
-// (novox/hq ADR 0184). The host looks again, after the moment in which that happens.
-func TestAServiceThatDiesJustAfterItsRestartIsNotReportedRestarted(t *testing.T) {
-	serviceSettle = 0
-	// Alive at the first look after starting, dead at the second — the shape of a daemon that
-	// refuses what it was just given.
-	started, looks := false, 0
-	run := func(ctx context.Context, name string, args ...string) (string, error) {
-		if args[0] == "show" {
-			state := "active"
-			if started {
-				if looks++; looks >= 2 {
-					state = "failed"
-				}
-			}
-			return "LoadState=loaded\nActiveState=" + state + "\n", nil
-		}
-		if args[0] == "start" {
-			started = true
-		}
-		return "", nil
-	}
-	dir := t.TempDir()
-	d := parse(t, `{"declaration":1,"resources":[
-	  {"id":"conf","type":"file","path":"`+dir+`/jail.conf","content":"[sshd]\n","mode":"0644"},
-	  {"id":"run","type":"service","unit":"fail2ban.service","state":"running","restart-on":["conf"]}
-	]}`)
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
-	if err == nil {
-		t.Fatal("a service that died just after being restarted was reported as restarted")
-	}
-	if !strings.Contains(err.Error(), "fail2ban.service") || !strings.Contains(err.Error(), "is stopped") {
-		t.Errorf("the failure does not name the unit and what it is now: %v", err)
-	}
-	if looks < 2 {
-		t.Errorf("the host looked at the unit %d time(s) after starting it; it must look again", looks)
-	}
-}
-
-// A unit still coming up reads as running at both looks and is accepted: the second look is for a
-// unit that WAS running and is not any more, never a wait for a slow one to finish starting.
-func TestAUnitStillStartingIsNotAFailure(t *testing.T) {
-	serviceSettle = 0
-	run := func(ctx context.Context, name string, args ...string) (string, error) {
-		if args[0] == "show" {
-			return "LoadState=loaded\nActiveState=activating\n", nil
-		}
-		return "", nil
-	}
-	d := parse(t, `{"declaration":1,"resources":[
-	  {"id":"s","type":"service","unit":"slow.service","state":"running"}
-	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
-		t.Fatalf("a unit still starting was reported as a failure: %v", err)
-	}
-}
-
-// And a service the declaration asks to be stopped is not waited on at all.
-func TestAServiceAskedToStopIsNotWaitedOn(t *testing.T) {
-	serviceSettle = 0
-	shows := 0
-	run := func(ctx context.Context, name string, args ...string) (string, error) {
-		if args[0] == "show" {
-			shows++
-			if shows == 1 {
-				return "LoadState=loaded\nActiveState=active\n", nil
-			}
-			return "LoadState=loaded\nActiveState=inactive\n", nil
-		}
-		return "", nil
-	}
-	d := parse(t, `{"declaration":1,"resources":[
-	  {"id":"s","type":"service","unit":"off.service","state":"stopped"}
-	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
-		t.Fatalf("stopping a service was reported as a failure: %v", err)
-	}
-}
-
-// The settle between a unit's two read-backs is a real pause on a machine and nothing in a test:
-// no test here drives a service manager that takes time, so paying it would only slow the suite
-// (novox/hq ADR 0184).
-func TestMain(m *testing.M) {
-	serviceSettle = 0
-	os.Exit(m.Run())
-}

internal/bootstrap/enrol.go

@@ -119,6 +119,11 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
 			// its ports and range on and not another that came up since.
 			args = append(args, "--tunnel", o.Tunnel)
 		}
+		// The enrolment account the token's secret is the password of exists in the mesh's records
+		// and nowhere on the bus yet (novox/hq 04-ISSUES/146): placed before the machine presents it.
+		if err := placeTheBusUsers(ctx, control, say); err != nil {
+			return out, err
+		}
 		joined, err := control.run(joining, o.Host, args...)
 		cancel()
 		if err != nil {
@@ -137,6 +142,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
 		}
 		out.Joined = true
 		say("  enrolled as       " + o.Node)
+		// And the node's own account, minted as it enrolled, before its agent connects as it.
+		if err := placeTheBusUsers(ctx, control, say); err != nil {
+			return out, err
+		}
 	}
 
 	// 4. The agent.
@@ -148,6 +157,40 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
 	return out, nil
 }
 
+// busAccounts and busContainer are where the installer's bundle raises the bus: the file its
+// configuration includes, and the container that reads it. The installer raised them, so it is the
+// one that knows them (examples/foundation-first-node-nats.lock).
+const (
+	busAccounts  = "/var/lib/mesh-bus-conf/accounts.conf"
+	busContainer = "mesh-broker"
+)
+
+// placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes
+// the bus re-read it.
+//
+// **Genesis's own step** (novox/hq 04-ISSUES/146). Every account on the bus reaches it in the
+// declaration of the machine that runs it — which needs that machine to be an enrolled node, and at
+// genesis it is not. The control plane composes the list and says it; whoever raised the bus places
+// it. That is this installer: it carried the bus in its bundle, so it knows where the bus reads it,
+// and the control plane never has to.
+func placeTheBusUsers(ctx context.Context, control controlPlane, say func(string)) error {
+	users, err := control.tell(ctx, "broker", "accounts")
+	if err != nil {
+		return fmt.Errorf("the control plane would not say the bus's users, so no machine could "+
+			"join it: %w", err)
+	}
+	if !strings.Contains(users, "accounts") {
+		return fmt.Errorf("the control plane's account of the bus's users is not one:\n%s", indent(users))
+	}
+	script := "umask 077 && cat > " + busAccounts + ".next <<'MESHBUSUSERS'\n" + users + "\nMESHBUSUSERS\n" +
+		"mv " + busAccounts + ".next " + busAccounts + " && docker kill -s HUP " + busContainer + " >/dev/null"
+	if out, err := control.run(ctx, "sh", "-c", script); err != nil {
+		return fmt.Errorf("the bus's users could not be placed at %s: %w\n%s", busAccounts, err, indent(out))
+	}
+	say("  bus users         placed")
+	return nil
+}
+
 // runTheHost makes sure something on this machine is listening to the mesh, and proves it.
 //
 // **The installer does not install the service, and says so.** A unit file is a packaging decision

internal/bootstrap/enrol_test.go

@@ -261,3 +261,56 @@ func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) {
 		t.Error("registry-mirror was not found")
 	}
 }
+
+// **Genesis places the bus's users, before the machine enrols and again after** (novox/hq
+// 04-ISSUES/146). The enrolment account exists only in the mesh's records until somebody writes it
+// beside the bus; so does the node's own, minted as it enrols. Without the first, the machine is
+// refused by the bus it just raised; without the second, its agent is.
+func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
+	enrolled := false
+	runtime := &asked{answer: func(name string, args []string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "node list"):
+			if enrolled {
+				return "anchor               here           01J0\n", nil
+			}
+			return "", nil
+		case strings.Contains(joined, "node add"):
+			return "added anchor\n", nil
+		case strings.Contains(joined, "token issue"):
+			return "a token for anchor, good once:\n\n  " + strings.Repeat("t", 240) + "\n\n", nil
+		case strings.Contains(joined, "broker accounts"):
+			return "accounts {\n  MESH { users = [] }\n}\n", nil
+		case name == "/usr/local/bin/mesh-host":
+			enrolled = true
+			return "enrolled as anchor\n", nil
+		case name == "pgrep", name == "sh":
+			return "", nil
+		}
+		return "", fmt.Errorf("unexpected: %s %v", name, args)
+	}}
+
+	if _, err := Enrol(context.Background(), Options{
+		Node: "anchor", State: filepath.Join(t.TempDir(), "state.json"), Timeout: time.Second,
+		Host: "/usr/local/bin/mesh-host", HostInBackground: true,
+	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
+		func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+
+	placed, enrol := []int{}, -1
+	for i, c := range runtime.commands {
+		if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") &&
+			strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") {
+			placed = append(placed, i)
+		}
+		if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
+			enrol = i
+		}
+	}
+	if enrol < 0 || len(placed) != 2 || placed[0] > enrol || placed[1] < enrol {
+		t.Fatalf("the bus's users were not placed before the machine enrolled and again after "+
+			"(placed at %v, enrolled at %d):\n%s", placed, enrol, strings.Join(runtime.commands, "\n"))
+	}
+}

internal/declaration/declaration.go

@@ -569,18 +569,6 @@ type Process struct {
 	// that runs once does not run on a schedule, and something that is not running cannot be
 	// restarted when a file changes.
 	Schedule string `json:"schedule,omitempty"`
-
-	// Replaces names resources this declaration no longer declares that this process takes the
-	// place of (novox/hq issue 213). Such a resource is not removed with the other orphans, before
-	// anything is applied: it is removed only once this process is applied and still running a
-	// moment later, and kept when it is not. So the thing being replaced answers until the thing
-	// replacing it does — the controller moving from its container to a process is the case: the
-	// container removed first left nothing answering the mesh's verbs for as long as fetching,
-	// unpacking and starting the process took, and for ever if the process did not start.
-	//
-	// For a process that stays up; a step or a scheduled run is not running a moment later by
-	// design, so there is nothing to hand over to.
-	Replaces []string `json:"replaces,omitempty"`
 }
 
 func (d *Process) Identity() string { return d.ID }
@@ -666,20 +654,6 @@ func (d *Process) validate(where string, _ bool) []string {
 			problems = append(problems, where+": "+err.Error())
 		}
 	}
-	if len(d.Replaces) > 0 && (d.RunOnce || d.Schedule != "") {
-		problems = append(problems, where+
-			": only a process that stays up replaces something — a step or a scheduled run is not "+
-			"running a moment later, so what it replaced would be removed with nothing in its place, "+
-			"or never")
-	}
-	for _, id := range d.Replaces {
-		switch {
-		case strings.TrimSpace(id) == "":
-			problems = append(problems, where+": replaces names an empty id")
-		case id == d.ID:
-			problems = append(problems, where+": a process cannot replace itself")
-		}
-	}
 	return problems
 }
 
@@ -896,7 +870,7 @@ type Package struct {
 	ID      string `json:"id"`
 	Type    Type   `json:"type"`
 	Package string `json:"package"`
-	// Absent declares that the package is NOT installed (novox/hq ADR 0180): the host removes it
+	// Absent declares that the package is NOT installed (novox/hq ADR 0175): the host removes it
 	// when it is, and leaves a machine that never had it alone. For the one case a module replaces
 	// software the machine was found with and the operator has decided it does not come back — the
 	// firewall front end a converged machine's filter module retired. Nothing to undo when the
@@ -1483,7 +1457,6 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 		d.Resources = append(d.Resources, resource)
 	}
 	problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
-	problems = append(problems, checkReplaces(d.Resources)...)
 	if env.Adoption == nil {
 		for _, r := range d.Resources {
 			if r.Kind() == TypeOpening {
@@ -1667,36 +1640,3 @@ func stripComments(raw []byte) []byte {
 	}
 	return []byte(strings.Join(kept, "\n"))
 }
-
-// checkReplaces refuses a `replaces` naming something this same declaration still declares, or
-// named by two processes (novox/hq issue 213). What is replaced is what the declaration no longer
-// says — a resource still declared is applied, not handed over, and one handed to two replacements
-// would go when the first of them came up, whatever became of the second.
-func checkReplaces(resources []Resource) []string {
-	declared := map[string]bool{}
-	for _, r := range resources {
-		declared[r.Identity()] = true
-	}
-	var problems []string
-	by := map[string]string{}
-	for _, r := range resources {
-		p, ok := r.(*Process)
-		if !ok {
-			continue
-		}
-		for _, id := range p.Replaces {
-			if declared[id] {
-				problems = append(problems, fmt.Sprintf(
-					"resource %q: replaces %q, which this declaration still declares — a process "+
-						"replaces what the declaration no longer says", p.ID, id))
-			}
-			if other, taken := by[id]; taken && other != p.ID {
-				problems = append(problems, fmt.Sprintf(
-					"resource %q: replaces %q, which %q replaces too — one thing has one replacement",
-					p.ID, id, other))
-			}
-			by[id] = p.ID
-		}
-	}
-	return problems
-}

internal/declaration/replaces_test.go

@@ -1,63 +0,0 @@
-package declaration
-
-import (
-	"strings"
-	"testing"
-)
-
-// novox/hq issue 213: a process says what it takes the place of, so the host can keep the old one
-// answering until the new one does. What it may name is narrow, and each refusal is said here.
-
-const aReplacingProcess = `{"id":"m.controller","type":"process","name":"m","source":"https://store.invalid/m",
-  "digest":"sha256:` + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + `","run":["./m","serve"]%s}`
-
-func replacing(extra string) string {
-	return `{"declaration":1,"resources":[` + strings.Replace(aReplacingProcess, "%s", extra, 1) + `]}`
-}
-
-func TestAProcessMayNameWhatItReplaces(t *testing.T) {
-	d, err := Parse([]byte(replacing(`,"replaces":["m.server"]`)))
-	if err != nil {
-		t.Fatalf("a process replacing an undeclared container was refused: %v", err)
-	}
-	if p := d.Resources[0].(*Process); len(p.Replaces) != 1 || p.Replaces[0] != "m.server" {
-		t.Fatalf("replaces was not read: %+v", p)
-	}
-}
-
-// What is replaced is what the declaration no longer says. A resource still declared is applied,
-// and handing it over as well would remove something the same declaration asks to keep.
-func TestAProcessMayNotReplaceSomethingStillDeclared(t *testing.T) {
-	raw := `{"declaration":1,"resources":[
-	  {"id":"m.server","type":"directory","path":"/tmp/x"},
-	  ` + strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + `]}`
-	if _, err := Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "still declares") {
-		t.Fatalf("a process replacing a declared resource was accepted: %v", err)
-	}
-}
-
-func TestOnlyAProcessThatStaysUpReplacesAnything(t *testing.T) {
-	for _, mode := range []string{`,"run-once":true`, `,"schedule":"0 3 * * *"`} {
-		if _, err := Parse([]byte(replacing(mode + `,"replaces":["m.server"]`))); err == nil {
-			t.Errorf("a process with %s was allowed to replace something", mode)
-		}
-	}
-}
-
-func TestAProcessCannotReplaceItselfOrNothing(t *testing.T) {
-	for _, bad := range []string{`,"replaces":["m.controller"]`, `,"replaces":[""]`} {
-		if _, err := Parse([]byte(replacing(bad))); err == nil {
-			t.Errorf("replaces %s was accepted", bad)
-		}
-	}
-}
-
-func TestOneThingHasOneReplacement(t *testing.T) {
-	second := strings.NewReplacer(`"id":"m.controller"`, `"id":"m.other"`, `"name":"m"`, `"name":"other"`).
-		Replace(strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1))
-	raw := `{"declaration":1,"resources":[` +
-		strings.Replace(aReplacingProcess, "%s", `,"replaces":["m.server"]`, 1) + "," + second + `]}`
-	if _, err := Parse([]byte(raw)); err == nil {
-		t.Fatal("two processes replacing one resource were accepted")
-	}
-}

internal/firewall/filters.go

@@ -179,18 +179,27 @@ func legacyFilters(rules, tool string, ufwActive bool) []Filter {
 			}
 		}
 	}
-	// A chain of refusals is a ban list when every refusal names the sources it refuses and the
-	// chain accepts nothing — the same rule the nftables side applies, and no more.
-	//
-	// **The policy of the chains that jump to it says nothing about what it is.** An earlier cut
-	// required every path into the chain to come from a built-in whose policy accepts, and the
-	// mesh's own intrusion prevention then read as a foreign rule set on the home server: its ban
-	// chain hangs off the container runtime's user chain as well as INPUT, and that machine's
-	// forward policy is DROP because the runtime set it. The machine reported "NOT the mesh alone"
-	// about a chain the mesh had just written (novox/hq ADR 0186). The policy is already classified
-	// where it belongs — as the runtime's — so requiring it here counted it twice.
+	var entered func(chain string, seen map[string]bool) bool
+	entered = func(chain string, seen map[string]bool) bool {
+		if seen[chain] || accepting[chain] || len(jumpedFrom[chain]) == 0 {
+			return false
+		}
+		seen[chain] = true
+		for _, from := range jumpedFrom[chain] {
+			if p, builtIn := policy[from]; builtIn {
+				if p != "ACCEPT" {
+					return false
+				}
+				continue
+			}
+			if !entered(from, seen) {
+				return false
+			}
+		}
+		return true
+	}
 	ban := func(chain, line string) bool {
-		return bansSources(line) && !accepting[chain] && len(jumpedFrom[chain]) > 0
+		return bansSources(line) && entered(chain, map[string]bool{})
 	}
 	type seen struct {
 		owner string
@@ -311,7 +320,7 @@ func Active(ctx context.Context, run Runner) bool {
 }
 
 // Installed says whether ufw is on this machine at all: a command that is not there is a front end
-// that was uninstalled (novox/hq ADR 0180), not one that is silent.
+// that was uninstalled (novox/hq ADR 0175), not one that is silent.
 func Installed(ctx context.Context, run Runner) bool {
 	_, err := run(ctx, "ufw", "status")
 	return !missing(err)
@@ -321,6 +330,6 @@ func Installed(ctx context.Context, run Runner) bool {
 const (
 	RetiredByMesh  = "mesh"
 	RetiredFoundSo = "found-inactive"
-	// RetiredRemoved is a front end uninstalled by the module that replaced it (ADR 0180).
+	// RetiredRemoved is a front end uninstalled by the module that replaced it (ADR 0175).
 	RetiredRemoved = "removed"
 )

internal/firewall/neighbour_ban_test.go

@@ -1,60 +0,0 @@
-package firewall
-
-import (
-	"os"
-	"testing"
-)
-
-// The mesh's own ban list is a ban wherever it hangs (novox/hq ADR 0186).
-//
-// Captured from the home server after the intrusion prevention had banned four addresses: its ban
-// chain is jumped to from INPUT, whose policy accepts, and from the container runtime's user chain,
-// which hangs off a FORWARD the runtime set to DROP. Requiring every path to come from an accepting
-// built-in made the machine report "NOT the mesh alone" about a chain the mesh had just written.
-func TestTheMeshsOwnBanChainIsABanBehindADroppingForward(t *testing.T) {
-	legacy, err := os.ReadFile("testdata/home-server-bans-S.txt")
-	if err != nil {
-		t.Fatal(err)
-	}
-	filters := Filters("", map[string]string{"iptables-legacy": string(legacy)}, false)
-
-	var ban, other []string
-	for _, f := range filters {
-		switch f.Owner {
-		case OwnerBan:
-			ban = append(ban, f.Where)
-		case OwnerOther:
-			other = append(other, f.Where)
-		}
-	}
-	if len(other) > 0 {
-		t.Errorf("the machine reports %v as rule sets the mesh did not write", other)
-	}
-	found := false
-	for _, w := range ban {
-		if w == "chain f2b-route-proxy (iptables-legacy)" {
-			found = true
-		}
-	}
-	if !found {
-		t.Errorf("the intrusion prevention's own chain was not read as a ban; bans were %v", ban)
-	}
-	if !Alone(filters) {
-		t.Error("a machine filtered by the mesh and its own bans does not read as the mesh alone")
-	}
-}
-
-// A chain that accepts anything is doing more than banning, and is still not a ban — which is what
-// keeps a predecessor's allow-and-drop chain classified as something the operator must look at.
-func TestAChainThatAcceptsIsNotABan(t *testing.T) {
-	rules := "-P INPUT ACCEPT\n" +
-		"-A INPUT -j HAL-MESH-ONLY\n" +
-		"-N HAL-MESH-ONLY\n" +
-		"-A HAL-MESH-ONLY -s 10.0.0.0/8 -j ACCEPT\n" +
-		"-A HAL-MESH-ONLY -s 203.0.113.7/32 -j DROP\n"
-	for _, f := range Filters("", map[string]string{"iptables-legacy": rules}, false) {
-		if f.Where == "chain HAL-MESH-ONLY (iptables-legacy)" && f.Owner != OwnerOther {
-			t.Errorf("a chain that accepts was classified as %s", f.Owner)
-		}
-	}
-}

internal/firewall/testdata/home-server-bans-S.txt

@@ -1,147 +0,0 @@
--P INPUT ACCEPT
--P FORWARD DROP
--P OUTPUT ACCEPT
--N DOCKER
--N DOCKER-BRIDGE
--N DOCKER-CT
--N DOCKER-FORWARD
--N DOCKER-INTERNAL
--N DOCKER-USER
--N f2b-route-proxy
--N ufw-after-forward
--N ufw-after-input
--N ufw-after-logging-forward
--N ufw-after-logging-input
--N ufw-after-logging-output
--N ufw-after-output
--N ufw-before-forward
--N ufw-before-input
--N ufw-before-logging-forward
--N ufw-before-logging-input
--N ufw-before-logging-output
--N ufw-before-output
--N ufw-reject-forward
--N ufw-reject-input
--N ufw-reject-output
--N ufw-track-forward
--N ufw-track-input
--N ufw-track-output
--A INPUT -p tcp -j f2b-route-proxy
--A INPUT -j ufw-before-logging-input
--A INPUT -j ufw-before-input
--A INPUT -j ufw-after-input
--A INPUT -j ufw-after-logging-input
--A INPUT -j ufw-reject-input
--A INPUT -j ufw-track-input
--A FORWARD -j DOCKER-USER
--A FORWARD -j DOCKER-FORWARD
--A FORWARD -j ufw-before-logging-forward
--A FORWARD -j ufw-before-forward
--A FORWARD -j ufw-after-forward
--A FORWARD -j ufw-after-logging-forward
--A FORWARD -j ufw-reject-forward
--A FORWARD -j ufw-track-forward
--A OUTPUT -j ufw-before-logging-output
--A OUTPUT -j ufw-before-output
--A OUTPUT -j ufw-after-output
--A OUTPUT -j ufw-after-logging-output
--A OUTPUT -j ufw-reject-output
--A OUTPUT -j ufw-track-output
--A DOCKER -d 172.17.0.18/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8686 -j ACCEPT
--A DOCKER -d 172.17.0.14/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8989 -j ACCEPT
--A DOCKER -d 172.17.0.15/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 7878 -j ACCEPT
--A DOCKER -d 172.17.0.5/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9117 -j ACCEPT
--A DOCKER -d 172.17.0.13/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6789 -j ACCEPT
--A DOCKER -d 172.19.0.2/32 ! -i br-32062158f584 -o br-32062158f584 -p tcp -m tcp --dport 8080 -j ACCEPT
--A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 5432 -j ACCEPT
--A DOCKER -d 172.27.0.2/32 ! -i br-0910a98c6158 -o br-0910a98c6158 -p tcp -m tcp --dport 5678 -j ACCEPT
--A DOCKER -d 172.17.0.21/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3579 -j ACCEPT
--A DOCKER -d 172.17.0.19/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8181 -j ACCEPT
--A DOCKER -d 172.17.0.17/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8787 -j ACCEPT
--A DOCKER -d 172.17.0.16/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6767 -j ACCEPT
--A DOCKER -d 172.17.0.12/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
--A DOCKER -d 172.17.0.11/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT
--A DOCKER -d 172.17.0.10/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9443 -j ACCEPT
--A DOCKER -d 172.17.0.10/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9000 -j ACCEPT
--A DOCKER -d 172.17.0.9/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
--A DOCKER -d 172.17.0.7/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 1880 -j ACCEPT
--A DOCKER -d 172.28.0.2/32 ! -i br-b11461b5b028 -o br-b11461b5b028 -p tcp -m tcp --dport 80 -j ACCEPT
--A DOCKER -d 172.23.0.14/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 6543 -j ACCEPT
--A DOCKER -d 172.23.0.14/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 5432 -j ACCEPT
--A DOCKER -d 172.23.0.5/32 ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -p tcp -m tcp --dport 8000 -j ACCEPT
--A DOCKER -d 172.26.0.3/32 ! -i br-b0fec361ccaa -o br-b0fec361ccaa -p tcp -m tcp --dport 6167 -j ACCEPT
--A DOCKER -d 172.26.0.2/32 ! -i br-b0fec361ccaa -o br-b0fec361ccaa -p tcp -m tcp --dport 80 -j ACCEPT
--A DOCKER -d 172.17.0.8/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8000 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10001 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8880 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8843 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8443 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8080 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6789 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 5514 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 3478 -j ACCEPT
--A DOCKER -d 172.17.0.6/32 ! -i docker0 -o docker0 -p udp -m udp --dport 1900 -j ACCEPT
--A DOCKER -d 172.25.0.3/32 ! -i br-b98821f7dc38 -o br-b98821f7dc38 -p tcp -m tcp --dport 8000 -j ACCEPT
--A DOCKER -d 172.18.0.3/32 ! -i br-442a0bfc65f8 -o br-442a0bfc65f8 -p tcp -m tcp --dport 1433 -j ACCEPT
--A DOCKER -d 172.20.0.3/32 ! -i br-afa37ac8b33d -o br-afa37ac8b33d -p tcp -m tcp --dport 8081 -j ACCEPT
--A DOCKER -d 172.20.0.3/32 ! -i br-afa37ac8b33d -o br-afa37ac8b33d -p tcp -m tcp --dport 1883 -j ACCEPT
--A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 8086 -j ACCEPT
--A DOCKER -d 172.21.0.2/32 ! -i br-df15d8e19ec7 -o br-df15d8e19ec7 -p tcp -m tcp --dport 6379 -j ACCEPT
--A DOCKER -d 172.30.0.3/32 ! -i br-521eab9a3a5e -o br-521eab9a3a5e -p tcp -m tcp --dport 8283 -j ACCEPT
--A DOCKER -d 172.17.0.3/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 3000 -j ACCEPT
--A DOCKER ! -i br-32062158f584 -o br-32062158f584 -j DROP
--A DOCKER ! -i docker0 -o docker0 -j DROP
--A DOCKER ! -i br-521eab9a3a5e -o br-521eab9a3a5e -j DROP
--A DOCKER ! -i br-df15d8e19ec7 -o br-df15d8e19ec7 -j DROP
--A DOCKER ! -i br-afa37ac8b33d -o br-afa37ac8b33d -j DROP
--A DOCKER ! -i br-442a0bfc65f8 -o br-442a0bfc65f8 -j DROP
--A DOCKER ! -i br-b98821f7dc38 -o br-b98821f7dc38 -j DROP
--A DOCKER ! -i br-b0fec361ccaa -o br-b0fec361ccaa -j DROP
--A DOCKER ! -i br-66ffa5c1cba5 -o br-66ffa5c1cba5 -j DROP
--A DOCKER ! -i br-b11461b5b028 -o br-b11461b5b028 -j DROP
--A DOCKER ! -i br-2df4e541b877 -o br-2df4e541b877 -j DROP
--A DOCKER ! -i br-0910a98c6158 -o br-0910a98c6158 -j DROP
--A DOCKER-BRIDGE -o br-32062158f584 -j DOCKER
--A DOCKER-BRIDGE -o docker0 -j DOCKER
--A DOCKER-BRIDGE -o br-521eab9a3a5e -j DOCKER
--A DOCKER-BRIDGE -o br-df15d8e19ec7 -j DOCKER
--A DOCKER-BRIDGE -o br-afa37ac8b33d -j DOCKER
--A DOCKER-BRIDGE -o br-442a0bfc65f8 -j DOCKER
--A DOCKER-BRIDGE -o br-b98821f7dc38 -j DOCKER
--A DOCKER-BRIDGE -o br-b0fec361ccaa -j DOCKER
--A DOCKER-BRIDGE -o br-66ffa5c1cba5 -j DOCKER
--A DOCKER-BRIDGE -o br-b11461b5b028 -j DOCKER
--A DOCKER-BRIDGE -o br-2df4e541b877 -j DOCKER
--A DOCKER-BRIDGE -o br-0910a98c6158 -j DOCKER
--A DOCKER-CT -o br-32062158f584 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-521eab9a3a5e -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-df15d8e19ec7 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-afa37ac8b33d -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-442a0bfc65f8 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-b98821f7dc38 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-b0fec361ccaa -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-66ffa5c1cba5 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-b11461b5b028 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-2df4e541b877 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-CT -o br-0910a98c6158 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-FORWARD -j DOCKER-CT
--A DOCKER-FORWARD -j DOCKER-INTERNAL
--A DOCKER-FORWARD -j DOCKER-BRIDGE
--A DOCKER-FORWARD -i br-32062158f584 -j ACCEPT
--A DOCKER-FORWARD -i docker0 -j ACCEPT
--A DOCKER-FORWARD -i br-521eab9a3a5e -j ACCEPT
--A DOCKER-FORWARD -i br-df15d8e19ec7 -j ACCEPT
--A DOCKER-FORWARD -i br-afa37ac8b33d -j ACCEPT
--A DOCKER-FORWARD -i br-442a0bfc65f8 -j ACCEPT
--A DOCKER-FORWARD -i br-b98821f7dc38 -j ACCEPT
--A DOCKER-FORWARD -i br-b0fec361ccaa -j ACCEPT
--A DOCKER-FORWARD -i br-66ffa5c1cba5 -j ACCEPT
--A DOCKER-FORWARD -i br-b11461b5b028 -j ACCEPT
--A DOCKER-FORWARD -i br-2df4e541b877 -j ACCEPT
--A DOCKER-FORWARD -i br-0910a98c6158 -j ACCEPT
--A DOCKER-USER -p tcp -j f2b-route-proxy
--A f2b-route-proxy -s 13.70.107.184/32 -j REJECT --reject-with icmp-port-unreachable
--A f2b-route-proxy -s 45.138.12.51/32 -j REJECT --reject-with icmp-port-unreachable
--A f2b-route-proxy -s 20.214.191.94/32 -j REJECT --reject-with icmp-port-unreachable
--A f2b-route-proxy -j RETURN

internal/identity/identity_test.go

@@ -409,3 +409,26 @@ func TestATokenSaysWhatTheMeshCallsThisMachine(t *testing.T) {
 		t.Fatalf("the name did not survive the token: %q", token.Node)
 	}
 }
+
+// A token through the tunnel carries the one peer, in the field names the control plane writes
+// (novox/hq ADR 0169), and an incomplete tunnel is refused naming what is missing.
+func TestATokenThroughTheTunnelParsesAndAPartOneIsRefused(t *testing.T) {
+	whole := map[string]any{"v": 1, "node": "n", "broker": "10.42.0.1:4222", "fingerprint": "sha256:x",
+		"signer": make([]byte, 32), "secret": "s",
+		"tunnel": map[string]any{"key": "k", "address": "10.42.0.9/32", "range": "10.42.0.0/16",
+			"hub_key": "h", "hub_endpoint": "198.51.100.1:51820"}}
+	raw, _ := json.Marshal(whole)
+	got, err := ParseToken(base64.RawURLEncoding.EncodeToString(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Tunnel == nil || got.Tunnel.HubEndpoint != "198.51.100.1:51820" || got.Tunnel.Range != "10.42.0.0/16" {
+		t.Fatalf("the tunnel was not read: %+v", got.Tunnel)
+	}
+	whole["tunnel"] = map[string]any{"key": "k"}
+	raw, _ = json.Marshal(whole)
+	if _, err := ParseToken(base64.RawURLEncoding.EncodeToString(raw)); err == nil ||
+		!strings.Contains(err.Error(), "the hub's tunnel key") {
+		t.Fatalf("a token with half a tunnel was taken: %v", err)
+	}
+}

internal/identity/overlay.go

@@ -5,6 +5,8 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"fmt"
+	"os"
+	"strings"
 )
 
 // The node's key on the private network, which is a different key from the one that says who it
@@ -64,6 +66,19 @@ func OverlayKeyFrom(privateBase64 string) (OverlayKey, error) {
 	}, nil
 }
 
+// LoadOverlayKey reads the key `key` made and left in its file (novox/hq ADR 0169).
+func LoadOverlayKey(path string) (OverlayKey, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return OverlayKey{}, err
+	}
+	key, err := OverlayKeyFrom(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return OverlayKey{}, fmt.Errorf("%s does not hold a tunnel key: %w", path, err)
+	}
+	return key, nil
+}
+
 // OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface
 // configuration rather than embedded in it.
 //

internal/identity/token.go

@@ -35,6 +35,21 @@ type Token struct {
 	// firewall found here before enrolling, because an adopted node keeps that firewall in force.
 	// Absent for a converged node.
 	Adopted bool `json:"adopted,omitempty"`
+
+	// Tunnel is this machine's first tunnel, when the token was issued for the key it made with
+	// `key` (novox/hq ADR 0169): its own address and the hub to reach. It brings the tunnel up from
+	// this alone and reaches the bus over it, so the bus never has to face the internet.
+	Tunnel *TokenTunnel `json:"tunnel,omitempty"`
+}
+
+// TokenTunnel is the joining machine's side of its first tunnel. Field names are the wire format
+// the control plane writes.
+type TokenTunnel struct {
+	Key         string `json:"key"`
+	Address     string `json:"address"`
+	Range       string `json:"range"`
+	HubKey      string `json:"hub_key"`
+	HubEndpoint string `json:"hub_endpoint"`
 }
 
 // ParseToken reads a token a person pasted.
@@ -70,6 +85,17 @@ func ParseToken(encoded string) (Token, error) {
 	if strings.TrimSpace(t.Secret) == "" {
 		missing = append(missing, "the one-time secret")
 	}
+	if tt := t.Tunnel; tt != nil {
+		for _, part := range []struct{ value, says string }{
+			{tt.Key, "the tunnel key it was issued for"}, {tt.Address, "this machine's address"},
+			{tt.Range, "the private network's range"}, {tt.HubKey, "the hub's tunnel key"},
+			{tt.HubEndpoint, "where the hub's tunnel is dialled"},
+		} {
+			if strings.TrimSpace(part.value) == "" {
+				missing = append(missing, part.says)
+			}
+		}
+	}
 	if len(missing) > 0 {
 		// Refused whole rather than used partially. A token missing the fingerprint would have
 		// this node connect to whatever answers at that address, and one missing the signing key

internal/system/arch.go

@@ -3,10 +3,7 @@ package system
 import (
 	"context"
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 )
@@ -59,119 +56,42 @@ func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	// **The package manager's own words, and a name for the case that looks like a bug in the
 	// declaration and is not.** A stale index asks the mirrors for a version they have already
 	// superseded and gets a 404 from every one of them — so the package exists, the declaration is
-	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
-	// old as the index fails one step later, on the signature of whatever a mirror still had.
-	//
-	// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
-	// the error rather than the output; this classifier read the output alone and so never saw a
-	// single "failed retrieving file", and the control node reported a ten-week-old database as
-	// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
+	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002).
 	//
 	// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
 	// libraries this machine does not have: a partial upgrade, which Arch does not support and
 	// which breaks the machine in a way that surfaces much later as something unrelated. The
 	// remedy is a full upgrade, and it is a decision about the whole machine rather than
-	// something to do silently in the middle of applying one resource. Whose decision, and on
-	// what schedule, is issue 205's question; until it is answered the host says what it sees.
-	said := strings.TrimSpace(out + "\n" + err.Error())
-	switch classifyInstallFailure(said) {
-	case installStale:
+	// something to do silently in the middle of applying one resource.
+	//
+	// So this says which of the two it is looking at. A declaration that is wrong and a machine
+	// that is out of date fail identically otherwise, and they are fixed in completely different
+	// places.
+	if staleIndex(out) {
 		return fmt.Errorf(
-			"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
-				"the package database on this machine is %s and the mirrors no longer serve what it "+
-				"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
-				"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
-				"host does not sync one package by itself, because on this distribution that is a "+
-				"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
-			name, syncDatabaseAge(), name, said)
-	case installMirrors:
-		return fmt.Errorf(
-			"no mirror could be reached to fetch %s, and the package database on this machine is "+
-				"%s: this reads as the mirrors or the network, not as this machine being out of "+
-				"date — try again when they answer.\n\n%s",
-			name, syncDatabaseAge(), said)
+			"%s could not be fetched from any mirror, which is what a stale package index looks "+
+				"like: this machine is asking for a version the mirrors have replaced. The "+
+				"package and the declaration are probably both fine. It is fixed by upgrading "+
+				"the machine, not by this host syncing one package — that would be a partial "+
+				"upgrade, which this distribution does not support.\n\n%s",
+			name, strings.TrimSpace(out))
 	}
 	return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
 }
 
-// How a failed install is read, from what the package manager said.
-type installFailure int
-
-const (
-	installOther installFailure = iota
-	// installStale: the machine's package database or keyring is older than what the mirrors
-	// serve — every mirror 404s the file the database names, or a package that did arrive fails
-	// its signature against a keyring that never saw the key.
-	installStale
-	// installMirrors: no mirror could be reached at all, and nothing says the database is old.
-	installMirrors
-)
-
-// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
-// code is the same for every one of these.
-func classifyInstallFailure(said string) installFailure {
-	lower := strings.ToLower(said)
-	gone := strings.Count(lower, "returned error: 404")
-	fetching := strings.Contains(lower, "failed retrieving file")
-	badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
-		strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
-		strings.Contains(lower, "is unknown trust") ||
-		strings.Contains(lower, "could not be looked up remotely")
-	switch {
-	case badSignature:
-		return installStale
-	case fetching && gone > 0:
-		// Every mirror, not one: a single mirror failing is an ordinary transient thing and
-		// retrying is the answer. pacman walks its whole mirror list before giving up, so more
-		// than one 404 among the lines is the index being old rather than one host being wrong.
-		if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
-			!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
-			return installStale
-		}
-		return installMirrors
-	case fetching:
-		return installMirrors
-	}
-	return installOther
-}
-
-// staleIndex is the yes-or-no form older callers and tests use.
-func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
-
-// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
-// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
-// install so a ten-week-old database is told apart from a mirror outage by reading one line.
+// staleIndex reports whether a failed install looks like the machine's view being old rather than
+// the package being wrong.
 //
-// A variable so a test can say what the machine's database looks like without having one.
-var syncDatabaseAge = func() string {
-	entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
-	if err != nil || len(entries) == 0 {
-		return "of unknown age (no sync database found under /var/lib/pacman/sync)"
+// By what the package manager said, because there is nothing else to go on: the exit code is the
+// same for both.
+func staleIndex(out string) bool {
+	said := strings.ToLower(out)
+	if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") {
+		return false
 	}
-	var newest time.Time
-	for _, e := range entries {
-		info, err := os.Stat(e)
-		if err == nil && info.ModTime().After(newest) {
-			newest = info.ModTime()
-		}
-	}
-	if newest.IsZero() {
-		return "of unknown age"
-	}
-	return describeAge(newest, time.Now())
-}
-
-// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
-func describeAge(when, now time.Time) string {
-	days := int(now.Sub(when).Hours() / 24)
-	span := fmt.Sprintf("%d days old", days)
-	switch {
-	case days < 1:
-		span = "less than a day old"
-	case days >= 14:
-		span = fmt.Sprintf("%d weeks old", days/7)
-	}
-	return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
+	// Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying
+	// is the answer; every one of them saying the file is gone is the index being old.
+	return strings.Contains(said, "error") || strings.Count(said, "404") > 1
 }
 
 // ServiceState reads what systemd says about a unit.

internal/system/arch_install_test.go

@@ -1,115 +0,0 @@
-package system
-
-import (
-	"context"
-	"errors"
-	"strings"
-	"testing"
-	"time"
-)
-
-// pacman's own words from the control node on 2026-10-02 (novox/hq 04-ISSUES/205): every mirror
-// 404s the versioned file a ten-week-old database names, and the one copy that arrives fails its
-// signature. Errors are pacman's stderr, which the runner folds into the error, not the output.
-const staleStderr = `error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.hetzner.com : The requested URL returned error: 404
-error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.rackspace.com : The requested URL returned error: 404
-error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from arch.lucassymons.net : Could not resolve host: arch.lucassymons.net
-warning: fatal error from arch.lucassymons.net, skipping for the remainder of this transaction
-error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirrors.cqu.edu.cn : Connection timed out after 10002 milliseconds
-error: nodejs: signature from "Bert Peters (packager key) <bertptrs@archlinux.org>" is invalid
-error: failed to commit transaction (invalid or corrupted package (PGP signature))`
-
-const staleStdout = `resolving dependencies...
-looking for conflicting packages...
-
-Packages (4) ada-3.4.4-1  c-ares-1.34.8-1  simdjson-1:4.6.4-1  nodejs-26.5.0-1
-
-:: Retrieving packages...
- nodejs-26.5.0-1-x86_64 downloading...
-checking keyring...
-checking package integrity...
-:: File /var/cache/pacman/pkg/nodejs-26.5.0-1-x86_64.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)).
-Errors occurred, no packages were upgraded.`
-
-// A runner that behaves as ExecRunner does on failure: stdout as the output, stderr in the error.
-func pacmanFailing(stdout, stderr string) Runner {
-	return func(_ context.Context, name string, args ...string) (string, error) {
-		return stdout, errors.New(name + " exited 1: " + stderr)
-	}
-}
-
-func TestAStaleDatabaseIsSaidAsOneWithItsAgeAndTheRemedy(t *testing.T) {
-	was := syncDatabaseAge
-	defer func() { syncDatabaseAge = was }()
-	syncDatabaseAge = func() string {
-		return describeAge(time.Date(2026, 7, 24, 16, 56, 0, 0, time.UTC), time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC))
-	}
-
-	err := arch{}.InstallPackage(context.Background(), pacmanFailing(staleStdout, staleStderr), "nodejs")
-	if err == nil {
-		t.Fatal("a failed install must fail")
-	}
-	for _, want := range []string{
-		"the package database on this machine is from 2026-07-24, 10 weeks old",
-		"stale package index",
-		"upgrading the machine — a full upgrade (`pacman -Syu`) by its operator — before the mesh can install nodejs",
-		"partial upgrade",
-		"returned error: 404", // pacman's own words follow
-	} {
-		if !strings.Contains(err.Error(), want) {
-			t.Errorf("the error does not say %q:\n%s", want, err)
-		}
-	}
-	if strings.Contains(err.Error(), "mirrors or the network") {
-		t.Errorf("a stale database must not be read as a mirror outage:\n%s", err)
-	}
-}
-
-// The same 404s read from stderr alone — the half this classifier used to be blind to.
-func TestTheClassifierReadsWhatPacmanWroteToStderr(t *testing.T) {
-	if classifyInstallFailure(staleStderr) != installStale {
-		t.Fatal("every mirror 404ing the named file is a stale database")
-	}
-	if classifyInstallFailure(staleStdout) != installStale {
-		t.Fatal("a corrupted-signature line alone is a stale keyring")
-	}
-	if classifyInstallFailure("") != installOther {
-		t.Fatal("nothing said is nothing classified")
-	}
-}
-
-func TestAnUnreachableMirrorWithAFreshDatabaseIsAMirrorProblem(t *testing.T) {
-	was := syncDatabaseAge
-	defer func() { syncDatabaseAge = was }()
-	syncDatabaseAge = func() string { return "from 2026-10-02, less than a day old" }
-	outage := `error: failed retrieving file 'core.db' from mirror.hetzner.com : Could not resolve host: mirror.hetzner.com
-error: failed retrieving file 'core.db' from mirror.rackspace.com : Connection timed out after 10001 milliseconds
-error: failed to synchronize all databases (failed to retrieve some files)`
-	if classifyInstallFailure(outage) != installMirrors {
-		t.Fatal("no mirror answering, no 404, no signature fault: the mirrors, not the machine")
-	}
-	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", outage), "nodejs")
-	if err == nil || !strings.Contains(err.Error(), "mirrors or the network") || !strings.Contains(err.Error(), "less than a day old") {
-		t.Errorf("a mirror outage is said as one, with the database's age beside it:\n%v", err)
-	}
-}
-
-func TestAFailureThatIsNeitherKeepsPacmansWords(t *testing.T) {
-	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", "error: target not found: nodejsx"), "nodejsx")
-	if err == nil || !strings.Contains(err.Error(), "target not found") || strings.Contains(err.Error(), "package database on this machine") {
-		t.Errorf("an unknown package is pacman's own error, not a stale database:\n%v", err)
-	}
-}
-
-func TestDescribeAge(t *testing.T) {
-	now := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
-	for when, want := range map[time.Time]string{
-		now.Add(-2 * time.Hour):       "less than a day old",
-		now.Add(-5 * 24 * time.Hour):  "5 days old",
-		now.Add(-71 * 24 * time.Hour): "10 weeks old",
-	} {
-		if got := describeAge(when, now); !strings.HasSuffix(got, want) {
-			t.Errorf("%s: got %q, want suffix %q", when, got, want)
-		}
-	}
-}

internal/system/system.go

@@ -52,7 +52,7 @@ type System interface {
 	PackageInstalled(ctx context.Context, run Runner, name string) (bool, error)
 	InstallPackage(ctx context.Context, run Runner, name string) error
 	// RemovePackage uninstalls one package, leaving its dependencies and anything the operator
-	// changed in its configuration where the package manager leaves them (novox/hq ADR 0180).
+	// changed in its configuration where the package manager leaves them (novox/hq ADR 0175).
 	RemovePackage(ctx context.Context, run Runner, name string) error
 
 	// ServiceState is "running" or "stopped". A unit that does not exist is an error, never