A joining machine dials the bus once the hub has answered its tunnel

Issuing the token sends the hub its new peer, and the hub applies it on
its own time; a bus dialled before then timed out naming the bus. The
first tunnel now waits for a handshake with the hub, and says so in the
tunnel's words when there is none (novox/hq ADR 0169).

cmd/mesh-host/join_tunnel.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/identity"
+)
+
+// Joining through the tunnel (novox/hq ADR 0169).
+//
+// **The bus is never open to the internet, so a joining machine reaches it over the tunnel.** It
+// makes its tunnel key first and prints the public half; the token is issued for that key, and the
+// hub is told the key before the token is shown; the token carries the one peer this machine needs.
+// So the tunnel can come up before the mesh has said anything else — the circle ADR 0004 broke by
+// carrying the bus's address in the token is broken here by carrying the hub's.
+
+// tunnelConfigPath and tunnelUnit are where the mesh's own declaration puts the private network, so
+// the first tunnel is the same interface and unit the mesh takes over, not a second one beside it.
+var (
+	tunnelConfigPath = "/etc/wireguard/mesh0.conf"
+	tunnelUnit       = "wg-quick@mesh0"
+	// lookPath finds WireGuard's tools; a variable so a test needs none installed.
+	lookPath = exec.LookPath
+)
+
+// keyCommand makes this machine's tunnel key, or reads the one it already made, and prints the
+// public half: what the token is issued for. Making it twice would be a token issued for a key the
+// machine no longer has, so an existing key is kept.
+func keyCommand(opts options) error {
+	path := identity.OverlayKeyPath(opts.state)
+	if key, err := identity.LoadOverlayKey(path); err == nil {
+		fmt.Println(key.Public)
+		return nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if _, err := os.Stat(identity.Path(opts.state)); err == nil {
+		return fmt.Errorf("this machine has joined already (%s), and its tunnel key is its own; "+
+			"there is no key to make", identity.Path(opts.state))
+	}
+	key, err := identity.GenerateOverlayKey()
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	if err := os.WriteFile(path, []byte(key.Private+"\n"), 0o600); err != nil {
+		return fmt.Errorf("cannot write this machine's tunnel key: %w", err)
+	}
+	fmt.Println(key.Public)
+	fmt.Fprintln(os.Stderr, "\nthis machine's tunnel key, made here; the private half stays in "+path+".\n"+
+		"Issue the token for it — `token issue --new <name> --overlay-key <the line above>` — and enrol with that token.")
+	return nil
+}
+
+// tunnelKeyFor is the key a token through the tunnel was issued for, read from where `key` left it.
+// Refused when there is none, or it is another: the hub knows only the key the token names.
+func tunnelKeyFor(t *identity.TokenTunnel, state string) (identity.OverlayKey, error) {
+	path := identity.OverlayKeyPath(state)
+	key, err := identity.LoadOverlayKey(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return identity.OverlayKey{}, fmt.Errorf("this token was issued for a tunnel key, and this " +
+			"machine has none: run `nox-mesh-host key` here first and issue the token for the key it prints")
+	}
+	if err != nil {
+		return identity.OverlayKey{}, err
+	}
+	if key.Public != t.Key {
+		return identity.OverlayKey{}, fmt.Errorf("this token was issued for the tunnel key %s, and this "+
+			"machine's is %s — it is another machine's token, or the key was made again; issue a new "+
+			"token for %s", t.Key, key.Public, key.Public)
+	}
+	return key, nil
+}
+
+// tunnelConfig is the first tunnel: this machine's address, and the hub as its one peer, reaching the
+// whole private network through it. The private key is set from its file, as the mesh's own
+// declaration does it, so the file holds no secret.
+func tunnelConfig(t *identity.TokenTunnel, keyPath string) string {
+	return fmt.Sprintf(`# Written by nox-mesh-host enrol: the one peer a joining machine needs (novox/hq ADR 0169).
+# The mesh's own declaration replaces this once the machine has joined.
+[Interface]
+Address = %s
+PostUp = wg set %%i private-key %s
+
+[Peer]
+PublicKey = %s
+Endpoint = %s
+AllowedIPs = %s
+PersistentKeepalive = 25
+`, t.Address, keyPath, t.HubKey, t.HubEndpoint, t.Range)
+}
+
+// bringTheTunnelUp writes the first tunnel and starts it, so the bus the token names can be reached.
+func bringTheTunnelUp(ctx context.Context, t *identity.TokenTunnel, keyPath string, run apply.Runner) error {
+	if _, err := lookPath("wg-quick"); err != nil {
+		return errors.New("joining through the tunnel needs WireGuard's tools on this machine " +
+			"(wireguard-tools), and wg-quick is not here")
+	}
+	if err := os.MkdirAll(filepath.Dir(tunnelConfigPath), 0o700); err != nil {
+		return err
+	}
+	if err := os.WriteFile(tunnelConfigPath, []byte(tunnelConfig(t, keyPath)), 0o600); err != nil {
+		return fmt.Errorf("cannot write the first tunnel: %w", err)
+	}
+	if out, err := run(ctx, "systemctl", "restart", tunnelUnit); err != nil {
+		return fmt.Errorf("the first tunnel would not start (%s): %v %s", tunnelUnit, err, strings.TrimSpace(out))
+	}
+	fmt.Printf("the tunnel to the hub is up: %s, through %s\n", t.Address, t.HubEndpoint)
+	return waitForTheHub(ctx, run, handshakeWithin)
+}
+
+// handshakeWithin is how long the hub has to answer the first tunnel. Issuing the token sent the hub
+// this machine as a peer; the hub applies that on its own time, and a bus dialled before it has is a
+// timeout that names the bus rather than the tunnel.
+var handshakeWithin = 90 * time.Second
+
+// waitForTheHub waits until the tunnel has shaken hands with the hub, so the bus is dialled over a
+// tunnel that answers — and says so in the tunnel's own words when it does not.
+func waitForTheHub(ctx context.Context, run apply.Runner, within time.Duration) error {
+	deadline := time.Now().Add(within)
+	for {
+		out, err := run(ctx, "wg", "show", "mesh0", "latest-handshakes")
+		if err == nil {
+			for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+				fields := strings.Fields(line)
+				if len(fields) == 2 && fields[1] != "0" {
+					fmt.Println("the hub answered the tunnel")
+					return nil
+				}
+			}
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("the hub has not answered the tunnel in %s: the token may be another machine's, "+
+				"the hub may not have been sent this machine as a peer, or its tunnel's port is not reachable "+
+				"from here", within)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+}

cmd/mesh-host/join_tunnel_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/identity"
+)
+
+// `key` makes the tunnel key once and prints its public half; asked again it prints the same one,
+// because a token may already have been issued for it (novox/hq ADR 0169).
+func TestKeyMakesTheTunnelKeyOnceAndKeepsIt(t *testing.T) {
+	dir := t.TempDir()
+	opts := options{state: filepath.Join(dir, "state.json")}
+	first := captureStdout(t, func() {
+		if err := keyCommand(opts); err != nil {
+			t.Fatal(err)
+		}
+	})
+	second := captureStdout(t, func() {
+		if err := keyCommand(opts); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if strings.TrimSpace(first) == "" || strings.TrimSpace(first) != strings.TrimSpace(second) {
+		t.Fatalf("the key changed between two asks: %q then %q", first, second)
+	}
+	info, err := os.Stat(identity.OverlayKeyPath(opts.state))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Mode().Perm() != 0o600 {
+		t.Errorf("the private half is readable beyond root: %v", info.Mode().Perm())
+	}
+}
+
+// A token through the tunnel takes the key it was issued for, and says so when this machine has none
+// or another.
+func TestATokenThroughTheTunnelTakesItsOwnKey(t *testing.T) {
+	dir := t.TempDir()
+	state := filepath.Join(dir, "state.json")
+	tt := &identity.TokenTunnel{Key: "x", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "h", HubEndpoint: "198.51.100.1:51820"}
+	if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "nox-mesh-host key") {
+		t.Fatalf("a machine with no key was not told to make one: %v", err)
+	}
+	captureStdout(t, func() { _ = keyCommand(options{state: state}) })
+	if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "issued for the tunnel key x") {
+		t.Fatalf("another machine's token was taken: %v", err)
+	}
+	mine, _ := identity.LoadOverlayKey(identity.OverlayKeyPath(state))
+	tt.Key = mine.Public
+	if got, err := tunnelKeyFor(tt, state); err != nil || got.Public != mine.Public {
+		t.Fatalf("this machine's own token was refused: %v", err)
+	}
+}
+
+// The first tunnel is the mesh's interface and unit, with the hub as its one peer and no secret in
+// the file — the same shape the mesh's declaration replaces it with.
+func TestTheFirstTunnelIsTheMeshsInterfaceWithTheHubAsItsPeer(t *testing.T) {
+	dir := t.TempDir()
+	tunnelConfigPath = filepath.Join(dir, "wireguard", "mesh0.conf")
+	lookPath = func(string) (string, error) { return "/usr/bin/wg-quick", nil }
+	t.Cleanup(func() { tunnelConfigPath = "/etc/wireguard/mesh0.conf" })
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "wg" {
+			return "HUBKEY\t1759400000\n", nil
+		}
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		return "", nil
+	}
+	tt := &identity.TokenTunnel{Key: "k", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "HUBKEY", HubEndpoint: "198.51.100.1:51820"}
+	captureStdout(t, func() {
+		if err := bringTheTunnelUp(context.Background(), tt, "/var/lib/mesh-host/overlay.key", run); err != nil {
+			t.Fatal(err)
+		}
+	})
+	raw, err := os.ReadFile(tunnelConfigPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := string(raw)
+	for _, want := range []string{"Address = 10.42.0.9/32", "PostUp = wg set %i private-key /var/lib/mesh-host/overlay.key",
+		"PublicKey = HUBKEY", "Endpoint = 198.51.100.1:51820", "AllowedIPs = 10.42.0.0/16", "PersistentKeepalive = 25"} {
+		if !strings.Contains(conf, want) {
+			t.Errorf("the first tunnel lacks %q:\n%s", want, conf)
+		}
+	}
+	if strings.Contains(conf, "PrivateKey") {
+		t.Error("the first tunnel's file holds the private key")
+	}
+	if len(ran) != 1 || ran[0] != "systemctl restart wg-quick@mesh0" {
+		t.Errorf("the tunnel was started as %v", ran)
+	}
+}
+
+// captureStdout is what fn printed to standard output.
+func captureStdout(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	was := os.Stdout
+	os.Stdout = w
+	fn()
+	os.Stdout = was
+	w.Close()
+	out, _ := io.ReadAll(r)
+	return string(out)
+}
+
+// The bus is dialled only once the hub has answered the tunnel, and a hub that never does is said
+// as the tunnel's fault rather than the bus's.
+func TestTheBusWaitsForTheHubToAnswer(t *testing.T) {
+	asked := 0
+	answersOnThird := func(_ context.Context, name string, args ...string) (string, error) {
+		asked++
+		if asked < 3 {
+			return "HUBKEY\t0\n", nil
+		}
+		return "HUBKEY\t1759400000\n", nil
+	}
+	captureStdout(t, func() {
+		if err := waitForTheHub(context.Background(), answersOnThird, time.Minute); err != nil {
+			t.Fatal(err)
+		}
+	})
+	never := func(context.Context, string, ...string) (string, error) { return "HUBKEY\t0\n", nil }
+	err := waitForTheHub(context.Background(), never, 0)
+	if err == nil || !strings.Contains(err.Error(), "has not answered the tunnel") {
+		t.Fatalf("a hub that never answered was not said: %v", err)
+	}
+}

cmd/mesh-host/main.go

@@ -96,6 +96,9 @@ const usage = `mesh-host — the node host
   reconcile   make this machine match what the mesh last told it — or, before any
               mesh has, the bundle this host carries
   bundle      show what this host carries
+  key         make this machine's tunnel key, or read the one it made, and print the public
+              half: what its join token is issued for (novox/hq ADR 0169)
+  enrol --token T  join the mesh — through the tunnel when the token was issued for a key
   overlay take  take over the tunnel found here (novox/hq ADR 0105): its key becomes this
               node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
   owned       what this host has applied and still owns
@@ -212,6 +215,9 @@ func parseArgs(args []string) (string, options, error) {
 func run(ctx context.Context, command string, opts options) error {
 	jsonOut, timeout := opts.json, opts.timeout
 	switch command {
+	case "key":
+		return keyCommand(opts)
+
 	case "profile":
 		p := profile.Detect(ctx, profile.Default(nil), timeout)
 		if jsonOut {
@@ -788,7 +794,18 @@ func enrol(ctx context.Context, opts options) error {
 			fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public)
 		}
 	}
-	if found == nil {
+	switch {
+	case found == nil && token.Tunnel != nil:
+		// Through the tunnel (novox/hq ADR 0169): the key `key` made, which the token names, and the
+		// tunnel brought up from the token before the bus is dialled — the bus is reached over it.
+		mine.Overlay, err = tunnelKeyFor(token.Tunnel, opts.state)
+		if err != nil {
+			return err
+		}
+		if err := bringTheTunnelUp(ctx, token.Tunnel, identity.OverlayKeyPath(opts.state), apply.ExecRunner); err != nil {
+			return err
+		}
+	case found == nil:
 		mine.Overlay, err = identity.GenerateOverlayKey()
 		if err != nil {
 			return err

internal/apply/apply.go

@@ -1407,6 +1407,25 @@ func applyPackage(ctx context.Context, sys system.System, r *declaration.Package
 	if err != nil {
 		return out, err
 	}
+	if r.Absent {
+		// Declared absent (novox/hq ADR 0175): removed when it is here, left alone when it is not.
+		if !installed {
+			out.Action = "unchanged"
+			out.Detail = "not installed, as declared"
+			return out, nil
+		}
+		if err := sys.RemovePackage(ctx, run, r.Package); err != nil {
+			return out, fmt.Errorf("removing %s: %w", r.Package, err)
+		}
+		if still, err := sys.PackageInstalled(ctx, run, r.Package); err != nil {
+			return out, err
+		} else if still {
+			return out, fmt.Errorf("%s was removed without error and the package database still has it", r.Package)
+		}
+		out.Action = "removed"
+		out.Detail = "declared absent; its configuration is left where the package manager leaves it"
+		return out, nil
+	}
 	if installed {
 		out.Action = "unchanged"
 		out.Detail = "already installed"
@@ -1583,6 +1602,15 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, n := range r.Networks {
 		b.WriteString("also-on " + n + "\n")
 	}
+	// And the capabilities it was granted (ADR 0170): one gained or dropped is a different
+	// container, and the runtime cannot change a running one's.
+	for _, c := range r.Capabilities {
+		b.WriteString("cap " + c + "\n")
+	}
+	// And where it logs (ADR 0179): the runtime cannot move a running container's output.
+	if r.Logging != "" {
+		b.WriteString("log " + r.Logging + "\n")
+	}
 	// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
 	// moves and the install is reported "updated" and re-established. Added only when present, so no
 	// ordinary container's or run-once step's digest moves for a field it does not set.
@@ -1777,6 +1805,14 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 	if r.Network != "" {
 		args = append(args, "--network", r.Network)
 	}
+	for _, c := range r.Capabilities {
+		args = append(args, "--cap-add", c)
+	}
+	if r.Logging != "" {
+		// The journal keeps the container's name on every line (CONTAINER_NAME), which is what a
+		// jail matches on (novox/hq ADR 0179); `docker logs` keeps working against the journal.
+		args = append(args, "--log-driver", r.Logging)
+	}
 	for _, d := range r.Dns {
 		args = append(args, "--dns", d)
 	}

internal/apply/left_out_test.go

@@ -134,3 +134,81 @@ func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) {
 		t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes)
 	}
 }
+
+// A container's capabilities reach the runtime and are part of its spec (novox/hq ADR 0170).
+func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name != "docker" {
+			return "", errors.New("not installed")
+		}
+		switch args[0] {
+		case "info":
+			return "29.0.0\n", nil
+		case "container":
+			return "false\t\n", errors.New("no such container")
+		case "run":
+			ran = args
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"fw","type":"container","name":"fw","image":"`+pinned+`","network":"host","capabilities":["NET_ADMIN"]}
+	]}`)
+	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	granted := false
+	for i, a := range ran {
+		if a == "--cap-add" && i+1 < len(ran) && ran[i+1] == "NET_ADMIN" {
+			granted = true
+		}
+	}
+	if !granted {
+		t.Fatalf("the capability was not granted: %v", ran)
+	}
+	with := d.Resources[0].(*declaration.Container)
+	without := *with
+	without.Capabilities = nil
+	if containerSpec(with, inputs{}) == containerSpec(&without, inputs{}) {
+		t.Fatal("a capability is not part of the container's spec")
+	}
+}
+
+// A package may be declared absent (novox/hq ADR 0175): removed when it is installed, read back,
+// left alone when it is not.
+func TestAPackageDeclaredAbsentIsRemovedWhenPresentAndLeftWhenNot(t *testing.T) {
+	installed := true
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if name != "pacman" {
+			return "", nil
+		}
+		switch args[0] {
+		case "-Q":
+			if args[1] == "pacman" || installed {
+				return args[1] + " 1.0\n", nil
+			}
+			return "", errors.New("package not found")
+		case "-R":
+			installed = false
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[{"id":"front-end","type":"package","package":"ufw","absent":true}]}`)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "removed" || !strings.Contains(strings.Join(ran, "\n"), "pacman -R --noconfirm ufw") {
+		t.Fatalf("an installed package declared absent was not removed: %+v\n%v", report.Outcomes[0], ran)
+	}
+	ran = nil
+	report, _, err = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "unchanged" || strings.Contains(strings.Join(ran, "\n"), "-R") {
+		t.Fatalf("a package already absent was touched: %+v\n%v", report.Outcomes[0], ran)
+	}
+}

internal/apply/logging_test.go

@@ -0,0 +1,43 @@
+package apply
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A container declared to log to the journal is run with the journal as its log driver, and the
+// place it logs is part of its spec, so moving it recreates the container (novox/hq ADR 0179).
+func TestAContainerLoggingToTheJournalIsRunThatWayAndRecreatedWhenMoved(t *testing.T) {
+	pinned := "postgres@sha256:" + strings.Repeat("a", 64)
+	var ran []string
+	run := func(_ context.Context, cmd string, args ...string) (string, error) {
+		if cmd == "docker" && len(args) > 0 && args[0] == "run" {
+			ran = args
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"front","type":"container","name":"front","image":"`+pinned+`","logging":"journald"}
+	]}`)
+	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	sent := false
+	for i, a := range ran {
+		if a == "--log-driver" && i+1 < len(ran) && ran[i+1] == "journald" {
+			sent = true
+		}
+	}
+	if !sent {
+		t.Fatalf("the container's output was not sent to the journal: %v", ran)
+	}
+	with := d.Resources[0].(*declaration.Container)
+	without := *with
+	without.Logging = ""
+	if containerSpec(with, inputs{}) == containerSpec(&without, inputs{}) {
+		t.Fatal("where a container logs is not part of its spec, so moving it would not recreate it")
+	}
+}

internal/apply/opening.go

@@ -76,6 +76,16 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri
 	if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive {
 		return "", nil
 	}
+	if !firewall.Installed(ctx, run) {
+		// Uninstalled (novox/hq ADR 0175): retired for good, by the module that replaced it. Said
+		// once, and nothing is asked of a command that is not there.
+		if rec.RetiredBy != firewall.RetiredRemoved {
+			rec.RetiredBy = firewall.RetiredRemoved
+			log("  the found firewall (ufw) is no longer installed; the mesh's filter is what filters this machine")
+			return "removed: ufw is no longer installed; the mesh's filter is what filters this machine", nil
+		}
+		return "", nil
+	}
 	active := firewall.Active(ctx, run)
 	if !active && !(rec.Forward != nil && !rec.DisabledByMesh) {
 		// Inactive, and either the mesh's doing already or nobody's recorded here: said as found,

internal/apply/opening_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
@@ -522,3 +523,33 @@ func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
 		t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall)
 	}
 }
+
+// A front end that is no longer installed is recorded as removed, said once, and asked nothing of
+// (novox/hq ADR 0175).
+func TestAnUninstalledFrontEndIsRetiredForGood(t *testing.T) {
+	dir := t.TempDir()
+	u := &ufwMachine{installed: false, ruleset: "table inet mesh\n"}
+	known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true,
+		RetiredBy: "mesh", FoundAt: time.Now()}}
+	converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
+	report, state, err := applyWith(t, converged, known, u.run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state.Firewall.RetiredBy != "removed" || !strings.Contains(report.Firewall, "no longer installed") {
+		t.Fatalf("record %+v, said %q", state.Firewall, report.Firewall)
+	}
+	u.asked = nil
+	report, _, err = applyWith(t, converged, state, u.run)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Firewall != "" {
+		t.Errorf("said again: %q", report.Firewall)
+	}
+	for _, a := range u.asked {
+		if strings.HasPrefix(a, "ufw") && a != "ufw status" {
+			t.Errorf("asked something of a front end that is not there: %v", u.asked)
+		}
+	}
+}

internal/bootstrap/enrol.go

@@ -119,6 +119,11 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
 			// its ports and range on and not another that came up since.
 			args = append(args, "--tunnel", o.Tunnel)
 		}
+		// The enrolment account the token's secret is the password of exists in the mesh's records
+		// and nowhere on the bus yet (novox/hq 04-ISSUES/146): placed before the machine presents it.
+		if err := placeTheBusUsers(ctx, control, say); err != nil {
+			return out, err
+		}
 		joined, err := control.run(joining, o.Host, args...)
 		cancel()
 		if err != nil {
@@ -137,6 +142,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
 		}
 		out.Joined = true
 		say("  enrolled as       " + o.Node)
+		// And the node's own account, minted as it enrolled, before its agent connects as it.
+		if err := placeTheBusUsers(ctx, control, say); err != nil {
+			return out, err
+		}
 	}
 
 	// 4. The agent.
@@ -148,6 +157,40 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
 	return out, nil
 }
 
+// busAccounts and busContainer are where the installer's bundle raises the bus: the file its
+// configuration includes, and the container that reads it. The installer raised them, so it is the
+// one that knows them (examples/foundation-first-node-nats.lock).
+const (
+	busAccounts  = "/var/lib/mesh-bus-conf/accounts.conf"
+	busContainer = "mesh-broker"
+)
+
+// placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes
+// the bus re-read it.
+//
+// **Genesis's own step** (novox/hq 04-ISSUES/146). Every account on the bus reaches it in the
+// declaration of the machine that runs it — which needs that machine to be an enrolled node, and at
+// genesis it is not. The control plane composes the list and says it; whoever raised the bus places
+// it. That is this installer: it carried the bus in its bundle, so it knows where the bus reads it,
+// and the control plane never has to.
+func placeTheBusUsers(ctx context.Context, control controlPlane, say func(string)) error {
+	users, err := control.tell(ctx, "broker", "accounts")
+	if err != nil {
+		return fmt.Errorf("the control plane would not say the bus's users, so no machine could "+
+			"join it: %w", err)
+	}
+	if !strings.Contains(users, "accounts") {
+		return fmt.Errorf("the control plane's account of the bus's users is not one:\n%s", indent(users))
+	}
+	script := "umask 077 && cat > " + busAccounts + ".next <<'MESHBUSUSERS'\n" + users + "\nMESHBUSUSERS\n" +
+		"mv " + busAccounts + ".next " + busAccounts + " && docker kill -s HUP " + busContainer + " >/dev/null"
+	if out, err := control.run(ctx, "sh", "-c", script); err != nil {
+		return fmt.Errorf("the bus's users could not be placed at %s: %w\n%s", busAccounts, err, indent(out))
+	}
+	say("  bus users         placed")
+	return nil
+}
+
 // runTheHost makes sure something on this machine is listening to the mesh, and proves it.
 //
 // **The installer does not install the service, and says so.** A unit file is a packaging decision

internal/bootstrap/enrol_test.go

@@ -261,3 +261,56 @@ func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) {
 		t.Error("registry-mirror was not found")
 	}
 }
+
+// **Genesis places the bus's users, before the machine enrols and again after** (novox/hq
+// 04-ISSUES/146). The enrolment account exists only in the mesh's records until somebody writes it
+// beside the bus; so does the node's own, minted as it enrols. Without the first, the machine is
+// refused by the bus it just raised; without the second, its agent is.
+func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
+	enrolled := false
+	runtime := &asked{answer: func(name string, args []string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "node list"):
+			if enrolled {
+				return "anchor               here           01J0\n", nil
+			}
+			return "", nil
+		case strings.Contains(joined, "node add"):
+			return "added anchor\n", nil
+		case strings.Contains(joined, "token issue"):
+			return "a token for anchor, good once:\n\n  " + strings.Repeat("t", 240) + "\n\n", nil
+		case strings.Contains(joined, "broker accounts"):
+			return "accounts {\n  MESH { users = [] }\n}\n", nil
+		case name == "/usr/local/bin/mesh-host":
+			enrolled = true
+			return "enrolled as anchor\n", nil
+		case name == "pgrep", name == "sh":
+			return "", nil
+		}
+		return "", fmt.Errorf("unexpected: %s %v", name, args)
+	}}
+
+	if _, err := Enrol(context.Background(), Options{
+		Node: "anchor", State: filepath.Join(t.TempDir(), "state.json"), Timeout: time.Second,
+		Host: "/usr/local/bin/mesh-host", HostInBackground: true,
+	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
+		func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+
+	placed, enrol := []int{}, -1
+	for i, c := range runtime.commands {
+		if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") &&
+			strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") {
+			placed = append(placed, i)
+		}
+		if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
+			enrol = i
+		}
+	}
+	if enrol < 0 || len(placed) != 2 || placed[0] > enrol || placed[1] < enrol {
+		t.Fatalf("the bus's users were not placed before the machine enrolled and again after "+
+			"(placed at %v, enrolled at %d):\n%s", placed, enrol, strings.Join(runtime.commands, "\n"))
+	}
+}

internal/declaration/declaration.go

@@ -870,6 +870,12 @@ type Package struct {
 	ID      string `json:"id"`
 	Type    Type   `json:"type"`
 	Package string `json:"package"`
+	// Absent declares that the package is NOT installed (novox/hq ADR 0175): the host removes it
+	// when it is, and leaves a machine that never had it alone. For the one case a module replaces
+	// software the machine was found with and the operator has decided it does not come back — the
+	// firewall front end a converged machine's filter module retired. Nothing to undo when the
+	// declaration drops it: the host does not install what a declaration stopped saying is absent.
+	Absent bool `json:"absent,omitempty"`
 }
 
 func (p *Package) Identity() string { return p.ID }
@@ -940,6 +946,20 @@ type Container struct {
 	// its siblings can name before any of them can resolve anything.
 	Dns []string `json:"dns,omitempty"`
 
+	// Capabilities are the Linux capabilities this container is granted beyond the runtime's
+	// default set, by name (novox/hq ADR 0170): a holder's runtime that changes the machine's packet
+	// filter asks for NET_ADMIN. Exactly these, named in the spec so a change recreates the
+	// container; a privileged container stays undeclarable.
+	Capabilities []string `json:"capabilities,omitempty"`
+
+	// Logging names where the runtime sends this container's output: "journald" sends it to the
+	// machine's journal, under the container's name, where what reads the machine's logs — its
+	// intrusion prevention first of all (novox/hq ADR 0179) — can read it the way it reads the
+	// machine's own services. Empty keeps the runtime's default, which is a file of the runtime's
+	// own that nothing but the runtime reads. Part of the spec: a container that logs elsewhere
+	// is a different container, and the runtime cannot change a running one's driver.
+	Logging string `json:"logging,omitempty"`
+
 	// Networks are networks this container also joins once created, by name — a found network a
 	// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
 	// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
@@ -1039,6 +1059,16 @@ func (c *Container) validate(where string, _ bool) []string {
 				"static address anywhere but a user-defined one")
 		}
 	}
+	for _, cap := range c.Capabilities {
+		if !capabilityName.MatchString(cap) {
+			problems = append(problems, where+": capabilities names "+strconv.Quote(cap)+", which is not a "+
+				"capability's name (CAP_NET_ADMIN or NET_ADMIN)")
+		}
+	}
+	if c.Logging != "" && c.Logging != "journald" {
+		problems = append(problems, where+": logging is "+strconv.Quote(c.Logging)+", and the only place a "+
+			"container's output can be sent besides the runtime's own file is \"journald\"")
+	}
 	for _, n := range c.Networks {
 		problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
 		if n == c.Network {
@@ -1209,6 +1239,10 @@ type Adoption struct {
 	Untaken map[string][]string `json:"untaken,omitempty"`
 }
 
+// capabilityName is what a Linux capability is called: upper case, underscores, an optional CAP_
+// prefix. The runtime accepts either spelling.
+var capabilityName = regexp.MustCompile(`^(CAP_)?[A-Z][A-Z0-9_]*$`)
+
 // AdoptionPrefix is the id prefix of what the mesh itself declares because a node is adopted —
 // its openings and its guard. Nothing under it belongs to a module, so none of it is ever held.
 const AdoptionPrefix = "adoption."

internal/declaration/declaration_test.go

@@ -504,3 +504,44 @@ func TestKeptNetworksAndLeftOutModulesAreReadStrictly(t *testing.T) {
 		t.Fatalf("a carried bundle leaving modules out was accepted: %v", err)
 	}
 }
+
+// A container may ask for a capability by name, and nothing else (novox/hq ADR 0170).
+func TestACapabilityIsNamedOrRefused(t *testing.T) {
+	image := "postgres@sha256:" + strings.Repeat("a", 64)
+	d, err := Parse([]byte(`{"declaration":1,"resources":[
+	  {"id":"fw","type":"container","name":"fw","image":"` + image + `","network":"host","capabilities":["NET_ADMIN","CAP_NET_RAW"]}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := d.Resources[0].(*Container).Capabilities; len(got) != 2 || got[0] != "NET_ADMIN" {
+		t.Fatalf("capabilities read as %v", got)
+	}
+	for _, bad := range []string{`"net_admin"`, `"ALL;rm -rf /"`, `"privileged"`} {
+		if _, err := Parse([]byte(`{"declaration":1,"resources":[
+		  {"id":"fw","type":"container","name":"fw","image":"` + image + `","capabilities":[` + bad + `]}]}`)); err == nil {
+			t.Errorf("%s was accepted as a capability", bad)
+		}
+	}
+}
+
+// A container may send its output to the machine's journal, and nowhere else but the runtime's own
+// file (novox/hq ADR 0179): what reads the machine's logs then reads the container's too.
+func TestAContainerMayLogToTheJournalAndNowhereElse(t *testing.T) {
+	image := "postgres@sha256:" + strings.Repeat("a", 64)
+	d, err := Parse([]byte(`{"declaration":1,"resources":[
+	  {"id":"front","type":"container","name":"front","image":"` + image + `","logging":"journald"}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := d.Resources[0].(*Container).Logging; got != "journald" {
+		t.Fatalf("logging read as %q", got)
+	}
+	for _, bad := range []string{`"syslog"`, `"none"`, `"json-file"`} {
+		if _, err := Parse([]byte(`{"declaration":1,"resources":[
+		  {"id":"front","type":"container","name":"front","image":"` + image + `","logging":` + bad + `}]}`)); err == nil {
+			t.Errorf("%s was accepted as a place to log", bad)
+		}
+	}
+}

internal/firewall/filters.go

@@ -319,8 +319,17 @@ func Active(ctx context.Context, run Runner) bool {
 	return err == nil && statusActive(out)
 }
 
+// Installed says whether ufw is on this machine at all: a command that is not there is a front end
+// that was uninstalled (novox/hq ADR 0175), not one that is silent.
+func Installed(ctx context.Context, run Runner) bool {
+	_, err := run(ctx, "ufw", "status")
+	return !missing(err)
+}
+
 // Retirements of a found firewall, as the host records them.
 const (
 	RetiredByMesh  = "mesh"
 	RetiredFoundSo = "found-inactive"
+	// RetiredRemoved is a front end uninstalled by the module that replaced it (ADR 0175).
+	RetiredRemoved = "removed"
 )

internal/identity/identity_test.go

@@ -409,3 +409,26 @@ func TestATokenSaysWhatTheMeshCallsThisMachine(t *testing.T) {
 		t.Fatalf("the name did not survive the token: %q", token.Node)
 	}
 }
+
+// A token through the tunnel carries the one peer, in the field names the control plane writes
+// (novox/hq ADR 0169), and an incomplete tunnel is refused naming what is missing.
+func TestATokenThroughTheTunnelParsesAndAPartOneIsRefused(t *testing.T) {
+	whole := map[string]any{"v": 1, "node": "n", "broker": "10.42.0.1:4222", "fingerprint": "sha256:x",
+		"signer": make([]byte, 32), "secret": "s",
+		"tunnel": map[string]any{"key": "k", "address": "10.42.0.9/32", "range": "10.42.0.0/16",
+			"hub_key": "h", "hub_endpoint": "198.51.100.1:51820"}}
+	raw, _ := json.Marshal(whole)
+	got, err := ParseToken(base64.RawURLEncoding.EncodeToString(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Tunnel == nil || got.Tunnel.HubEndpoint != "198.51.100.1:51820" || got.Tunnel.Range != "10.42.0.0/16" {
+		t.Fatalf("the tunnel was not read: %+v", got.Tunnel)
+	}
+	whole["tunnel"] = map[string]any{"key": "k"}
+	raw, _ = json.Marshal(whole)
+	if _, err := ParseToken(base64.RawURLEncoding.EncodeToString(raw)); err == nil ||
+		!strings.Contains(err.Error(), "the hub's tunnel key") {
+		t.Fatalf("a token with half a tunnel was taken: %v", err)
+	}
+}

internal/identity/overlay.go

@@ -5,6 +5,8 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"fmt"
+	"os"
+	"strings"
 )
 
 // The node's key on the private network, which is a different key from the one that says who it
@@ -64,6 +66,19 @@ func OverlayKeyFrom(privateBase64 string) (OverlayKey, error) {
 	}, nil
 }
 
+// LoadOverlayKey reads the key `key` made and left in its file (novox/hq ADR 0169).
+func LoadOverlayKey(path string) (OverlayKey, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return OverlayKey{}, err
+	}
+	key, err := OverlayKeyFrom(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return OverlayKey{}, fmt.Errorf("%s does not hold a tunnel key: %w", path, err)
+	}
+	return key, nil
+}
+
 // OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface
 // configuration rather than embedded in it.
 //

internal/identity/token.go

@@ -35,6 +35,21 @@ type Token struct {
 	// firewall found here before enrolling, because an adopted node keeps that firewall in force.
 	// Absent for a converged node.
 	Adopted bool `json:"adopted,omitempty"`
+
+	// Tunnel is this machine's first tunnel, when the token was issued for the key it made with
+	// `key` (novox/hq ADR 0169): its own address and the hub to reach. It brings the tunnel up from
+	// this alone and reaches the bus over it, so the bus never has to face the internet.
+	Tunnel *TokenTunnel `json:"tunnel,omitempty"`
+}
+
+// TokenTunnel is the joining machine's side of its first tunnel. Field names are the wire format
+// the control plane writes.
+type TokenTunnel struct {
+	Key         string `json:"key"`
+	Address     string `json:"address"`
+	Range       string `json:"range"`
+	HubKey      string `json:"hub_key"`
+	HubEndpoint string `json:"hub_endpoint"`
 }
 
 // ParseToken reads a token a person pasted.
@@ -70,6 +85,17 @@ func ParseToken(encoded string) (Token, error) {
 	if strings.TrimSpace(t.Secret) == "" {
 		missing = append(missing, "the one-time secret")
 	}
+	if tt := t.Tunnel; tt != nil {
+		for _, part := range []struct{ value, says string }{
+			{tt.Key, "the tunnel key it was issued for"}, {tt.Address, "this machine's address"},
+			{tt.Range, "the private network's range"}, {tt.HubKey, "the hub's tunnel key"},
+			{tt.HubEndpoint, "where the hub's tunnel is dialled"},
+		} {
+			if strings.TrimSpace(part.value) == "" {
+				missing = append(missing, part.says)
+			}
+		}
+	}
 	if len(missing) > 0 {
 		// Refused whole rather than used partially. A token missing the fingerprint would have
 		// this node connect to whatever answers at that address, and one missing the signing key

internal/profile/detectors.go

@@ -15,6 +15,9 @@ const (
 	CapServiceManager   = "service-manager"
 	CapFirewall         = "firewall"
 	CapOverlay          = "overlay"
+	// CapVirtualisation is a running virtualisation daemon: what the lab raises its machines on
+	// (novox/hq ADR 0172), and what grants a module the daemon's socket.
+	CapVirtualisation   = "virtualisation"
 	CapGraphicalSession = "graphical-session"
 	// CapSeat is hardware: somewhere a display server COULD run. CapGraphicalSession above is
 	// state: whether one IS running. Assignment needs the first.
@@ -205,6 +208,11 @@ func Default(runner Runner) []Detector {
 			why:    "lists the ruleset — needs the tool AND the privilege to use it",
 			runner: runner,
 		},
+		commandCapability{
+			name: CapVirtualisation, command: "incus", args: []string{"info"},
+			why:    "asks the virtualisation daemon about itself — a running daemon, not an installed client",
+			runner: runner,
+		},
 		commandCapability{
 			name: CapOverlay, command: "wg", args: []string{"show", "interfaces"},
 			why:    "asks the kernel for interfaces — needs the module, not just the tool",

internal/system/alpine.go

@@ -46,6 +46,11 @@ func (a alpine) PackageInstalled(ctx context.Context, run Runner, name string) (
 	return strings.TrimSpace(out) != "", nil
 }
 
+func (alpine) RemovePackage(ctx context.Context, run Runner, name string) error {
+	_, err := run(ctx, "apk", "del", name)
+	return err
+}
+
 func (alpine) InstallPackage(ctx context.Context, run Runner, name string) error {
 	_, err := run(ctx, "apk", "add", "--no-cache", name)
 	return err

internal/system/android.go

@@ -65,6 +65,10 @@ func (a android) InstallPackage(context.Context, Runner, string) error {
 	return fmt.Errorf("%w: package", ErrUnsupported)
 }
 
+func (a android) RemovePackage(context.Context, Runner, string) error {
+	return fmt.Errorf("%w: package", ErrUnsupported)
+}
+
 func (a android) ServiceState(context.Context, Runner, string) (string, error) {
 	return "", fmt.Errorf("%w: service (init is not reachable without root)", ErrUnsupported)
 }

internal/system/arch.go

@@ -39,6 +39,14 @@ func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bo
 	return true, nil
 }
 
+// RemovePackage removes one package and nothing it depends on: `-R`, not `-Rs`, because what else
+// relied on a dependency is not this declaration's to know. pacman keeps a configuration file the
+// operator changed as `.pacsave`, which is what "never flushed" comes to once the front end is gone.
+func (arch) RemovePackage(ctx context.Context, run Runner, name string) error {
+	_, err := run(ctx, "pacman", "-R", "--noconfirm", name)
+	return err
+}
+
 func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name)
 	if err == nil {

internal/system/system.go

@@ -51,6 +51,9 @@ type System interface {
 
 	PackageInstalled(ctx context.Context, run Runner, name string) (bool, error)
 	InstallPackage(ctx context.Context, run Runner, name string) error
+	// RemovePackage uninstalls one package, leaving its dependencies and anything the operator
+	// changed in its configuration where the package manager leaves them (novox/hq ADR 0175).
+	RemovePackage(ctx context.Context, run Runner, name string) error
 
 	// ServiceState is "running" or "stopped". A unit that does not exist is an error, never
 	// "stopped" — reporting absence as satisfaction is the fault this host exists to prevent.