Merge pull request 'Issue 339 follow-ups: a directory used as found is said and can be handed over; the runtime's data and any .ssh refused' (#64) from fix/339-found-directories-are-said into main

cmd/mesh-host/found.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A directory used as found is never quiet (novox/hq issue 339).
+//
+// The engine leaves the owner and mode of a directory it did not make: changing them would be root handing a
+// directory it never made to whichever account a declaration named. That is safe and, said nowhere, a module
+// whose data directory it cannot write would simply fail. So each such directory is said three ways: a WARNING
+// line in every apply's log; a resource of its module of kind `directory`, stated unhealthy on every look,
+// which the controller raises as that module's condition naming the directory and its owner; and a way out at
+// the machine's terminal, `mesh-host hand-over <path>`, after which the next apply gives it the declared owner
+// and mode like any directory of the mesh's.
+
+// foundState is where the node's state is, for the looks to read the directories used as found from; empty
+// until the engine runs.
+var foundState atomic.Value
+
+// handedOverFrom has the apply read the directories handed over from beside the state at statePath.
+func handedOverFrom(statePath string) {
+	apply.HandedOver = func() map[string]bool {
+		all, err := store.LoadHandedOver(statePath)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: %v; no directory counts as handed over until it reads\n", err)
+			return nil
+		}
+		out := map[string]bool{}
+		for _, h := range all {
+			out[apply.HandOverKey(h.ID, h.Path)] = true
+		}
+		return out
+	}
+}
+
+// foundDirectories is every directory the node's state records as used as found, sorted by id.
+func foundDirectories(known store.State) []store.Applied {
+	var out []store.Applied
+	for _, r := range known.Resources {
+		if r.Type == "directory" && r.AsFound {
+			out = append(out, r)
+		}
+	}
+	sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
+	return out
+}
+
+// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
+// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and how to hand
+// it over. The path is the target, which the controller keeps as evidence.
+func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health {
+	for _, r := range found {
+		at := strings.LastIndex(r.ID, ".")
+		if at <= 0 {
+			continue
+		}
+		owner, mode := "unknown", "unknown"
+		if info, err := os.Lstat(r.Target); err == nil {
+			mode = fmt.Sprintf("%o", info.Mode().Perm())
+			if uid, gid, ok := apply.OwnerOf(info); ok {
+				owner = fmt.Sprintf("%d:%d", uid, gid)
+			}
+		}
+		declared := r.Declared
+		if declared == "" {
+			declared = "another owner or mode"
+		}
+		h.Resources = append(h.Resources, link.ResourceHealth{Module: r.ID[:at], Resource: r.ID,
+			Kind: link.KindDirectory, Target: r.Target, State: link.StateUnhealthy,
+			Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+
+				"`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode,
+				declared),
+			Since: foundSince(r).UTC()})
+	}
+	return h
+}
+
+// foundNow is the directories used as found, read from the node's state; none when it cannot be read, which
+// the apply says on its own.
+func foundNow() []store.Applied {
+	path, _ := foundState.Load().(string)
+	if path == "" {
+		return nil
+	}
+	known, err := store.Load(path)
+	if err != nil {
+		return nil
+	}
+	return foundDirectories(known)
+}
+
+// handOverCommand hands one directory used as found to the mesh: run by a person, as root, at the machine.
+// It changes nothing on disk itself; the next apply gives the directory its declared owner and mode, and says so.
+func handOverCommand(opts options) error {
+	path := filepath.Clean(opts.file)
+	if !filepath.IsAbs(path) {
+		return fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one", opts.file)
+	}
+	known, err := store.Load(opts.state)
+	if err != nil {
+		return err
+	}
+	for _, r := range foundDirectories(known) {
+		if filepath.Clean(r.Target) != path {
+			continue
+		}
+		by := "a person at the machine"
+		if u, err := user.Current(); err == nil {
+			by = u.Username
+		}
+		if err := store.AddHandedOver(opts.state, store.HandOver{ID: r.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
+			return err
+		}
+		fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+
+			"then on — removed when empty and no longer declared\n", path, r.ID, orUnsaid(r.Declared))
+		return nil
+	}
+	return errors.New(path + " is not a directory this machine uses as found (`mesh-host owned` lists what the " +
+		"engine applied); nothing was handed over")
+}
+
+// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one
+// a send found from one found before it. A record from before that was kept says when it was last applied.
+func foundSince(r store.Applied) time.Time {
+	if !r.FoundAt.IsZero() {
+		return r.FoundAt
+	}
+	return r.AppliedAt
+}
+
+func orUnsaid(s string) string {
+	if s == "" {
+		return "the declared owner and mode"
+	}
+	return s
+}

cmd/mesh-host/found_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on
+// every statement (novox/hq issue 339): the controller raises that as the module's condition.
+func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
+	dir := t.TempDir()
+	known := store.State{Resources: []store.Applied{
+		{ID: "notes.data", Type: "directory", Target: dir, AsFound: true, Declared: "root, mode 755", AppliedAt: time.Now()},
+		{ID: "notes.cache", Type: "directory", Target: dir},
+	}}
+	h := withFoundDirectories(&link.Health{}, foundDirectories(known))
+	if len(h.Resources) != 1 {
+		t.Fatalf("stated: %+v", h.Resources)
+	}
+	r := h.Resources[0]
+	if r.Module != "notes" || r.Kind != link.KindDirectory || r.State != link.StateUnhealthy || r.Target != dir ||
+		!strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) || !strings.Contains(r.Reason, "owned by ") ||
+		!strings.Contains(r.Reason, "root, mode 755") || !strings.Contains(r.Reason, "hand-over") {
+		t.Fatalf("stated as: %+v", r)
+	}
+}
+
+// `mesh-host hand-over` records a directory used as found, and refuses one that is not.
+func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) {
+	dir := t.TempDir()
+	statePath := filepath.Join(dir, "state.json")
+	found := filepath.Join(dir, "found")
+	if err := store.Save(statePath, store.State{Resources: []store.Applied{
+		{ID: "notes.data", Type: "directory", Target: found, AsFound: true, Declared: "root, mode 755"},
+		{ID: "notes.mine", Type: "directory", Target: filepath.Join(dir, "mine")},
+	}}); err != nil {
+		t.Fatal(err)
+	}
+	var out bytes.Buffer
+	if err := handOverCommand(options{state: statePath, file: filepath.Join(dir, "mine"), out: &out}); err == nil {
+		t.Fatal("a directory the mesh made was handed over")
+	}
+	if err := handOverCommand(options{state: statePath, file: found + "/", out: &out}); err != nil {
+		t.Fatal(err)
+	}
+	all, err := store.LoadHandedOver(statePath)
+	if err != nil || len(all) != 1 || all[0].Path != found {
+		t.Fatalf("recorded: %+v %v", all, err)
+	}
+	handedOverFrom(statePath)
+	if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] {
+		t.Fatal("the apply does not see the hand-over")
+	}
+	if !strings.Contains(out.String(), "root, mode 755") {
+		t.Fatalf("said: %s", out.String())
+	}
+	_ = os.Remove(statePath)
+}

cmd/mesh-host/main.go

@@ -108,6 +108,7 @@ const usage = `mesh-host — the node host
   overlay take  take over the tunnel found here (novox/hq ADR 0105): its key becomes this
               node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
   owned       what this host has applied and still owns
+  hand-over <dir>  hand a directory used as found to the mesh: the next apply gives it its declared owner and mode
   version
 
   --json      machine-readable output
@@ -203,6 +204,13 @@ func parseArgs(args []string) (string, options, error) {
 		opts.file = positionals[0]
 		return command, opts, nil
 	}
+	if command == "hand-over" {
+		if len(positionals) != 1 {
+			return "", opts, errors.New("hand-over <directory>: the path of one directory used as found")
+		}
+		opts.file = positionals[0]
+		return command, opts, nil
+	}
 	if command == "overlay" {
 		if len(positionals) != 1 {
 			return "", opts, errors.New("overlay take [--tunnel <iface>]")
@@ -220,7 +228,13 @@ func parseArgs(args []string) (string, options, error) {
 
 func run(ctx context.Context, command string, opts options) error {
 	jsonOut, timeout := opts.json, opts.timeout
+	// Every apply this process makes counts what a person handed over, and every look says what is used as
+	// found (novox/hq issue 339).
+	handedOverFrom(opts.state)
+	foundState.Store(opts.state)
 	switch command {
+	case "hand-over":
+		return handOverCommand(opts)
 	case "key":
 		return keyCommand(opts)
 
@@ -1537,9 +1551,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
 			}
 			spaced = sp
 		}
+		// Every directory used as found, read from the node's state on every look (novox/hq issue 339).
+		found := foundNow()
 		since := time.Since(lastSaid)
 		healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
-			(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy())
+			(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0
 		if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
 			continue
 		}
@@ -1551,7 +1567,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
 				}
 			}
 		}
-		if queue.SayHealth(ctx, *withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt)) {
+		if queue.SayHealth(ctx, *withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
+			accountSt), found)) {
 			lastSaid, owed = time.Now(), false
 		}
 	}
@@ -1896,7 +1913,8 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 			as, _ := a.Look(ctx)
 			accountSt = &as
 		}
-		report.Health = withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt)
+		report.Health = withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt),
+			foundDirectories(updated))
 	}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,

internal/apply/apply.go

@@ -75,8 +75,11 @@ type Outcome struct {
 	groups []string
 	// unpacked is, for an archive, what it put on the machine (novox/hq issue 162).
 	unpacked *store.Unpacked
-	// asFound is, for a directory, that it is used as it was found (novox/hq issue 339).
-	asFound bool
+	// asFound is, for a directory, that it is used as it was found (novox/hq issue 339), and asFoundDeclared
+	// the owner and mode that were declared and not given it.
+	asFound         bool
+	asFoundDeclared string
+	asFoundAt       time.Time
 	// reads is, for a container, the digest of each file it was created reading, by path — so
 	// the next apply can say which one changed (novox/hq 04-ISSUES/103).
 	reads map[string]string
@@ -737,6 +740,8 @@ func ApplyMindingWindows(
 			Groups:    outcome.groups,
 			Unpacked:  outcome.unpacked,
 			AsFound:   outcome.asFound,
+			Declared:  outcome.asFoundDeclared,
+			FoundAt:   outcome.asFoundAt,
 			Holds:     holds(resource),
 		})
 		if outcome.found != nil {
@@ -769,6 +774,11 @@ func ApplyMindingWindows(
 			}
 		}
 		report.Outcomes = append(report.Outcomes, outcome)
+		if outcome.asFound {
+			// Never quiet (novox/hq issue 339): said on every apply, and stated unhealthy on every look until a
+			// person hands it over (FoundDirectories, `mesh-host hand-over`).
+			log(fmt.Sprintf("  WARNING %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
+		}
 		if handed {
 			// Recorded in its place, so the record going is forgotten now — and only now: a file whose
 			// new owner failed or was skipped keeps the old record, and the next apply hands it over.
@@ -1021,6 +1031,11 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
 	}
 	if existed && wasBefore {
 		ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound
+		// Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the
+		// declared owner and mode below, as any directory of the mesh's.
+		if !ours && HandedOver()[handOverKey(r.ID, r.Path)] {
+			ours = true
+		}
 		if !ours {
 			owned, err := ownedBy(r.Path, r.Owner)
 			if err != nil {
@@ -1032,9 +1047,14 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
 				if owner == "" {
 					owner = "root"
 				}
-				out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner and mode %o are "+
-					"left, though %s and %o were declared (novox/hq issue 339); set them by hand to hand it to the mesh",
-					before.Mode().Perm(), owner, mode.Perm())
+				out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm())
+				out.asFoundAt = time.Now().UTC()
+				if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() {
+					out.asFoundAt = previous.FoundAt
+				}
+				out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+
+					"left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+
+					"hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path)
 				return out, nil
 			}
 		}
@@ -3273,3 +3293,23 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) {
 	return "kept", fmt.Sprintf("no longer declared, and it held more than the mesh wrote — moved aside to %s; "+
 		"remove it by hand once you know what it is", aside), nil
 }
+
+// HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by
+// handOverKey: a found directory there is the mesh's, given its declared owner and mode at the next apply
+// (novox/hq issue 339). Set by the engine to read the file beside its state; a test sets its own.
+var HandedOver = func() map[string]bool { return nil }
+
+// HandOverKey is one hand-over: the resource it was for and the path. A hand-over is for that resource alone,
+// so it lapses if another declaration later points another resource at the same path.
+func HandOverKey(id, path string) string { return id + " " + filepath.Clean(path) }
+
+func handOverKey(id, path string) string { return HandOverKey(id, path) }
+
+// ownerText is a directory's owner as numbers, "uid:gid", or "unknown".
+func ownerText(info os.FileInfo) string {
+	uid, gid, ok := ownerOf(info)
+	if !ok {
+		return "unknown"
+	}
+	return fmt.Sprintf("%d:%d", uid, gid)
+}

internal/apply/found_is_said_test.go

@@ -0,0 +1,138 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A directory used as found is never quiet, and a person at the machine can hand it over (novox/hq issue 339).
+func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) {
+	dir := t.TempDir()
+	found := filepath.Join(dir, "found")
+	if err := os.Mkdir(found, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}`
+	var said []string
+	log := func(line string) { said = append(said, line) }
+	_, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
+		noServices, log, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") {
+		t.Fatalf("the apply did not warn of a directory used as found: %q", said)
+	}
+	if a, _ := state.Find("notes.data"); !a.AsFound || !strings.Contains(a.Declared, "755") {
+		t.Fatalf("the record does not say what was declared: %+v", a)
+	}
+	// Said again at the next apply: a warning once is a warning missed.
+	said = nil
+	if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared,
+		noServices, log, nil); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") {
+		t.Fatalf("the next apply did not warn again: %q", said)
+	}
+
+	// Handed over: the next apply gives it the declared mode and it is the mesh's.
+	before := HandedOver
+	HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} }
+	t.Cleanup(func() { HandedOver = before })
+	if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared,
+		noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if info, _ := os.Stat(found); info.Mode().Perm() != 0o755 {
+		t.Fatalf("a directory handed over was not given its declared mode: %o", info.Mode().Perm())
+	}
+	if a, _ := state.Find("notes.data"); a.AsFound {
+		t.Fatal("a directory handed over is still recorded as found")
+	}
+}
+
+// The review's additions to the guard: the spool, /opt, the container runtimes' data wherever the runtime keeps
+// it, and an account's .ssh as an access or a mount, or as a directory anywhere but its own home.
+func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) {
+	cfg := filepath.Join(t.TempDir(), "daemon.json")
+	if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	before := runtimeConfigs
+	runtimeConfigs = runtimeFiles{daemonJSON: cfg}
+	t.Cleanup(func() { runtimeConfigs = before })
+	for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
+		for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x",
+			"/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} {
+			if err := refusePath(path, kind, "notes", ""); err == nil {
+				t.Errorf("%v at %s was taken", kind, path)
+			}
+		}
+	}
+	for _, path := range []string{"/srv/backup/.ssh", "/srv/backup/.ssh/keys"} {
+		for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
+			if err := refusePath(path, kind, "notes", ""); err == nil {
+				t.Errorf("%v at %s was taken", kind, path)
+			}
+		}
+	}
+	if err := refusePath("/storage/dockerish", placingAccess, "notes", ""); err != nil {
+		t.Errorf("a path beside the data root: %v", err)
+	}
+}
+
+// When a directory was found is kept from the first apply that found it (novox/hq issue 339): the controller
+// tells a directory a send found from one found before it by that time, so it must not move with every apply.
+func TestWhenADirectoryWasFoundIsKept(t *testing.T) {
+	dir := t.TempDir()
+	found := filepath.Join(dir, "found")
+	if err := os.Mkdir(found, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}`
+	_, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
+		noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	first, _ := state.Find("notes.data")
+	if first.FoundAt.IsZero() {
+		t.Fatal("no time the directory was found")
+	}
+	time.Sleep(10 * time.Millisecond)
+	_, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again, _ := state.Find("notes.data"); !again.FoundAt.Equal(first.FoundAt) {
+		t.Fatalf("found at %v, then %v: the time moved with the apply", first.FoundAt, again.FoundAt)
+	}
+}
+
+// A hand-over is for the resource that was used as found there: another declaration that later points another
+// resource at the same path is not handed anything (novox/hq issue 339).
+func TestAHandOverIsForTheResourceThatAskedForIt(t *testing.T) {
+	dir := t.TempDir()
+	found := filepath.Join(dir, "found")
+	if err := os.Mkdir(found, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	before := HandedOver
+	HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} }
+	t.Cleanup(func() { HandedOver = before })
+	other := `{"id":"other.data","type":"directory","path":"` + found + `","mode":"0755"}`
+	if _, _, err := Apply(context.Background(), archHost(t), declare(t, other), store.State{}, store.OriginDeclared,
+		noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if info, _ := os.Stat(found); info.Mode().Perm() != 0o700 {
+		t.Fatalf("a hand-over for notes.data gave other.data the directory: mode %o", info.Mode().Perm())
+	}
+}

internal/apply/placement_guard.go

@@ -13,9 +13,14 @@ package apply
 //  1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc,
 //     /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc
 //     or /var/lib, never the root itself; owning one is owning everything in it.
-//  2. **Nothing is placed in /proc, /sys, /dev, /boot or /root, nor in the node-engine's own trees** (its
-//     state, its identity, its installed builds) but by its own module; nothing is mounted from /boot, /root or
-//     the engine's trees. A mount of a kernel file, a device or the clock is the plumbing systemPath names.
+//  2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data
+//     (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves
+//     them: runtimeDataRoots), nor in the node-engine's
+//     own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from
+//     those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing
+//     systemPath names.
+//  2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home,
+//     as that account's (rule 4).
 //  3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read
 //     as root's word; a directory there owned by another account is that account writing root's word. An
 //     access is never there at all: the operator's data is not the machine's configuration.
@@ -32,10 +37,12 @@ package apply
 
 import (
 	"bufio"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -52,15 +59,88 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l
 // forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's
 // home. engineTrees are the node-engine's own, which only its own module places in.
 var (
-	forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root"}
+	forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
+		"/var/lib/containers", "/var/lib/containerd"}
 	// forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is
 	// the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing.
-	forbiddenBelowMount = []string{"/boot", "/root"}
-	engineTrees         = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
+	forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers",
+		"/var/lib/containerd"}
+	engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
 	// rootsOnly are trees a directory below is root's, and an access is never in.
 	rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"}
 )
 
+// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its
+// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph), and podman's
+// storage.conf (graphroot). containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a
+// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the
+// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its
+// account. A variable so a test names its own.
+type runtimeFiles struct {
+	daemonJSON  string
+	units       []string
+	dropInDirs  []string
+	storageConf string
+}
+
+var runtimeConfigs = runtimeFiles{
+	daemonJSON:  "/etc/docker/daemon.json",
+	units:       []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"},
+	dropInDirs:  []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"},
+	storageConf: "/etc/containers/storage.conf",
+}
+
+var (
+	dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)("?)(/[^\s"]+)`)
+	graphRoot    = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*"(/[^"]+)"`)
+)
+
+// runtimeDataRoots is every place the container runtimes are configured to keep their data, beyond the default
+// trees forbiddenBelow names: every container's filesystem is there.
+func runtimeDataRoots() []string {
+	var out []string
+	add := func(p string) {
+		if filepath.IsAbs(p) {
+			out = append(out, filepath.Clean(p))
+		}
+	}
+	if raw, err := os.ReadFile(runtimeConfigs.daemonJSON); err == nil {
+		var c struct {
+			DataRoot string `json:"data-root"`
+			Graph    string `json:"graph"`
+		}
+		if json.Unmarshal(raw, &c) == nil {
+			add(c.DataRoot)
+			add(c.Graph)
+		}
+	}
+	units := append([]string(nil), runtimeConfigs.units...)
+	for _, dir := range runtimeConfigs.dropInDirs {
+		matches, _ := filepath.Glob(filepath.Join(dir, "*.conf"))
+		units = append(units, matches...)
+	}
+	for _, u := range units {
+		raw, err := os.ReadFile(u)
+		if err != nil {
+			continue
+		}
+		for _, line := range strings.Split(string(raw), "\n") {
+			if !strings.HasPrefix(strings.TrimSpace(line), "ExecStart") {
+				continue
+			}
+			for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) {
+				add(m[2])
+			}
+		}
+	}
+	if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil {
+		for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) {
+			add(m[1])
+		}
+	}
+	return out
+}
+
 // engineModule is the module whose resources may place in the engine's own trees.
 const engineModule = "mesh-host"
 
@@ -193,12 +273,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
 				"and owning or mounting it would be owning or mounting everything in it"}
 		}
 	}
-	trees := forbiddenBelow
+	trees := append([]string(nil), forbiddenBelow...)
 	if kind == placingMount {
 		// A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null
 		// (systemPath) — and what a setting can mount at all is a directory or an access, refused above it.
-		trees = forbiddenBelowMount
+		trees = append([]string(nil), forbiddenBelowMount...)
 	}
+	// The container runtimes' data, wherever the machine keeps it: every container's filesystem is there.
+	trees = append(trees, runtimeDataRoots()...)
 	for _, tree := range trees {
 		if atOrBelow(path, tree) {
 			return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
@@ -225,6 +307,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
 				"one is declared %s's", tree, owner)}
 		}
 	}
+	ssh := false
+	for _, part := range strings.Split(path, "/") {
+		ssh = ssh || part == ".ssh"
+	}
+	if ssh && kind != placingDirectory {
+		return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " +
+			"log in as it, and is never an access or a mount"}
+	}
 	if kind != placingDirectory {
 		return nil
 	}
@@ -235,6 +325,10 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
 			deepest = home
 		}
 	}
+	if ssh && deepest == "" {
+		return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " +
+			"as that account's"}
+	}
 	if deepest != "" {
 		if a := homes[deepest]; !ownedByAccount(owner, a) {
 			if owner == "" {

internal/apply/runtime_roots_test.go

@@ -0,0 +1,44 @@
+package apply
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// The container runtimes keep every container's filesystem somewhere the machine may have moved: dockerd's
+// --data-root on its unit's command line or a drop-in, podman's graphroot, containerd's own tree (novox/hq issue
+// 339). Each is refused as a place, an access or a mount, wherever it is said.
+func TestEveryRuntimesDataRootIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	write := func(name, content string) string {
+		t.Helper()
+		p := filepath.Join(dir, name)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		return p
+	}
+	unit := write("lib/docker.service", "[Service]\nExecStart=/usr/bin/dockerd -H fd:// --data-root /data/docker-a\n")
+	dropIn := write("etc/docker.service.d/root.conf", "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --data-root=/data/docker-b --log-level warn\n")
+	storage := write("containers/storage.conf", "[storage]\ndriver = \"overlay\"\n  graphroot = \"/data/podman\"\nrunroot = \"/run/containers/storage\"\n")
+	before := runtimeConfigs
+	runtimeConfigs = runtimeFiles{daemonJSON: filepath.Join(dir, "none.json"), units: []string{unit},
+		dropInDirs: []string{filepath.Dir(dropIn)}, storageConf: storage}
+	t.Cleanup(func() { runtimeConfigs = before })
+
+	for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
+		for _, path := range []string{"/var/lib/containerd", "/var/lib/containerd/io.containerd.content.v1",
+			"/data/docker-a/volumes", "/data/docker-b", "/data/podman/overlay"} {
+			if err := refusePath(path, kind, "notes", ""); err == nil {
+				t.Errorf("%v at %s was taken", kind, path)
+			}
+		}
+		if err := refusePath("/data/media", kind, "notes", ""); err != nil {
+			t.Errorf("%v at /data/media: %v", kind, err)
+		}
+	}
+}

internal/apply/ssh_home_test.go

@@ -0,0 +1,36 @@
+package apply
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// An account's .ssh below its own home, as that account's, is still placed (the ssh-client module's); as an
+// access or a mount it is refused there too (novox/hq issue 339).
+func TestAnAccountsOwnSSHIsPlacedAndNeverMounted(t *testing.T) {
+	dir := t.TempDir()
+	home := filepath.Join(dir, "home", "alice")
+	passwd := filepath.Join(dir, "passwd")
+	if err := os.WriteFile(passwd, fmt.Appendf(nil, "alice:x:4242:4242::%s:/bin/sh\n", home), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	before := passwdFile
+	passwdFile = passwd
+	t.Cleanup(func() { passwdFile = before })
+	if err := refusePath(home+"/.ssh", placingDirectory, "ssh-client", "alice"); err != nil {
+		t.Fatalf("alice's own .ssh: %v", err)
+	}
+	if err := refusePath(home+"/.ssh/config.d", placingDirectory, "ssh-client", "alice"); err != nil {
+		t.Fatalf("alice's own .ssh/config.d: %v", err)
+	}
+	for _, kind := range []placing{placingAccess, placingMount} {
+		if err := refusePath(home+"/.ssh", kind, "notes", ""); err == nil {
+			t.Errorf("%v of alice's .ssh was taken", kind)
+		}
+	}
+	if err := refusePath(home+"/.ssh", placingDirectory, "notes", "4243"); err == nil {
+		t.Error("another account's directory at alice's .ssh was taken")
+	}
+}

internal/apply/user.go

@@ -456,3 +456,6 @@ func ownAll(root, owner string) error {
 func ownerOf(info os.FileInfo) (uid, gid int, ok bool) {
 	return statOwner(info)
 }
+
+// OwnerOf is a file's numeric owner, as the engine reads it.
+func OwnerOf(info os.FileInfo) (uid, gid int, ok bool) { return statOwner(info) }

internal/link/messages.go

@@ -260,6 +260,14 @@ const KindUnit = "unit"
 // reason then starting "relogin needed".
 const KindAccount = "account"
 
+// KindDirectory is a directory a module declares that the engine uses as found (novox/hq issue 339): there before
+// the mesh, with another owner or mode than declared, and left so. Said unhealthy on every look, its reason
+// starting ReasonUsedAsFound, until a person hands it to the mesh at the machine (`mesh-host hand-over`).
+const KindDirectory = "directory"
+
+// ReasonUsedAsFound starts the reason of a directory used as found.
+const ReasonUsedAsFound = "used as found:"
+
 // UnitsHealth is the machine's service managers in one statement (issue 315).
 type UnitsHealth struct {
 	// State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no

internal/store/handed_over.go

@@ -0,0 +1,87 @@
+package store
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+// HandedOverName is the file, beside the node's state, of the directories a person handed to the mesh at the
+// machine (novox/hq issue 339). Its own file rather than a field of the state: the engine saves its state at
+// the end of every apply, from what it loaded at the start, and a hand-over written into the state during an
+// apply would be lost to that save.
+const HandedOverName = "handed-over.json"
+
+// HandedOverPath is that file beside the state at statePath.
+func HandedOverPath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), HandedOverName)
+}
+
+// HandOver is one directory a person handed to the mesh.
+type HandOver struct {
+	// ID is the resource that was used as found there: the hand-over is for it alone, and lapses if another
+	// resource later points at the path.
+	ID   string    `json:"id"`
+	Path string    `json:"path"`
+	At   time.Time `json:"at"`
+	// By is who ran the command, as the machine named them.
+	By string `json:"by,omitempty"`
+}
+
+// LoadHandedOver reads every hand-over. None is an empty list; a file that cannot be read is an error.
+func LoadHandedOver(statePath string) ([]HandOver, error) {
+	raw, err := os.ReadFile(HandedOverPath(statePath))
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading the directories handed to the mesh: %w", err)
+	}
+	var out []HandOver
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return nil, fmt.Errorf("the directories handed to the mesh (%s) are unreadable: %w", HandedOverPath(statePath), err)
+	}
+	return out, nil
+}
+
+// AddHandedOver records a hand-over, once per path, atomically.
+func AddHandedOver(statePath string, h HandOver) error {
+	all, err := LoadHandedOver(statePath)
+	if err != nil {
+		return err
+	}
+	for _, had := range all {
+		if had.Path == h.Path && had.ID == h.ID {
+			return nil
+		}
+	}
+	all = append(all, h)
+	sort.Slice(all, func(a, b int) bool { return all[a].Path < all[b].Path })
+	raw, err := json.MarshalIndent(all, "", "  ")
+	if err != nil {
+		return err
+	}
+	path := HandedOverPath(statePath)
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".handed-over-*.json")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if _, err := tmp.Write(append(raw, '\n')); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

internal/store/store.go

@@ -135,6 +135,11 @@ type Applied struct {
 	// declared owner and mode (novox/hq issue 339): the mesh uses it as found, never changing its owner or
 	// mode, and never removes it. Absent on a record from before: such a directory is the mesh's.
 	AsFound bool `json:"as_found,omitempty"`
+	// Declared is, for a directory used as found, the owner and mode that were declared and not given it.
+	Declared string `json:"declared,omitempty"`
+	// FoundAt is, for a directory used as found, when an apply first found it so: kept across applies, so the
+	// controller can tell a directory a send found from one found before it.
+	FoundAt time.Time `json:"found_at,omitempty"`
 
 	// Reads is, for a container, the digest of each file it was created reading — its env-files
 	// and the files mounted into it — by path (novox/hq 04-ISSUES/103).