Read every runtime's data root, keep when a directory was found, and key a hand-over to its resource

The review of #64 asked for containerd's tree, dockerd's --data-root from its
unit and drop-ins, and podman's graphroot to be refused like docker's own
(hq issue 339); and the controller now tells a directory a send found from
one found before it, which needs the time it was first found kept across
applies instead of the time of the last apply. A hand-over now names the
resource it was for, so it lapses if another declaration points at the path.

cmd/mesh-host/found.go

@@ -40,7 +40,7 @@ func handedOverFrom(statePath string) {
 		}
 		out := map[string]bool{}
 		for _, h := range all {
-			out[filepath.Clean(h.Path)] = true
+			out[apply.HandOverKey(h.ID, h.Path)] = true
 		}
 		return out
 	}
@@ -83,7 +83,7 @@ func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health {
 			Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+
 				"`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode,
 				declared),
-			Since: r.AppliedAt.UTC()})
+			Since: foundSince(r).UTC()})
 	}
 	return h
 }
@@ -121,7 +121,7 @@ func handOverCommand(opts options) error {
 		if u, err := user.Current(); err == nil {
 			by = u.Username
 		}
-		if err := store.AddHandedOver(opts.state, store.HandOver{Path: path, At: time.Now().UTC(), By: by}); err != nil {
+		if err := store.AddHandedOver(opts.state, store.HandOver{ID: r.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
 			return err
 		}
 		fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+
@@ -132,6 +132,15 @@ func handOverCommand(opts options) error {
 		"engine applied); nothing was handed over")
 }
 
+// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one
+// a send found from one found before it. A record from before that was kept says when it was last applied.
+func foundSince(r store.Applied) time.Time {
+	if !r.FoundAt.IsZero() {
+		return r.FoundAt
+	}
+	return r.AppliedAt
+}
+
 func orUnsaid(s string) string {
 	if s == "" {
 		return "the declared owner and mode"

cmd/mesh-host/found_test.go

@@ -56,7 +56,7 @@ func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) {
 		t.Fatalf("recorded: %+v %v", all, err)
 	}
 	handedOverFrom(statePath)
-	if !apply.HandedOver()[found] {
+	if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] {
 		t.Fatal("the apply does not see the hand-over")
 	}
 	if !strings.Contains(out.String(), "root, mode 755") {

internal/apply/apply.go

@@ -79,6 +79,7 @@ type Outcome struct {
 	// the owner and mode that were declared and not given it.
 	asFound         bool
 	asFoundDeclared string
+	asFoundAt       time.Time
 	// reads is, for a container, the digest of each file it was created reading, by path — so
 	// the next apply can say which one changed (novox/hq 04-ISSUES/103).
 	reads map[string]string
@@ -740,6 +741,7 @@ func ApplyMindingWindows(
 			Unpacked:  outcome.unpacked,
 			AsFound:   outcome.asFound,
 			Declared:  outcome.asFoundDeclared,
+			FoundAt:   outcome.asFoundAt,
 			Holds:     holds(resource),
 		})
 		if outcome.found != nil {
@@ -1031,7 +1033,7 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
 		ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound
 		// Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the
 		// declared owner and mode below, as any directory of the mesh's.
-		if !ours && HandedOver()[filepath.Clean(r.Path)] {
+		if !ours && HandedOver()[handOverKey(r.ID, r.Path)] {
 			ours = true
 		}
 		if !ours {
@@ -1046,6 +1048,10 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
 					owner = "root"
 				}
 				out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm())
+				out.asFoundAt = time.Now().UTC()
+				if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() {
+					out.asFoundAt = previous.FoundAt
+				}
 				out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+
 					"left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+
 					"hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path)
@@ -3289,10 +3295,16 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) {
 }
 
 // HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by
-// path: a found directory there is the mesh's, given its declared owner and mode at the next apply (novox/hq
-// issue 339). Set by the engine to read the file beside its state; a test sets its own.
+// handOverKey: a found directory there is the mesh's, given its declared owner and mode at the next apply
+// (novox/hq issue 339). Set by the engine to read the file beside its state; a test sets its own.
 var HandedOver = func() map[string]bool { return nil }
 
+// HandOverKey is one hand-over: the resource it was for and the path. A hand-over is for that resource alone,
+// so it lapses if another declaration later points another resource at the same path.
+func HandOverKey(id, path string) string { return id + " " + filepath.Clean(path) }
+
+func handOverKey(id, path string) string { return HandOverKey(id, path) }
+
 // ownerText is a directory's owner as numbers, "uid:gid", or "unknown".
 func ownerText(info os.FileInfo) string {
 	uid, gid, ok := ownerOf(info)

internal/apply/found_is_said_test.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/novox/mesh-host/internal/store"
 )
@@ -43,7 +44,7 @@ func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) {
 
 	// Handed over: the next apply gives it the declared mode and it is the mesh's.
 	before := HandedOver
-	HandedOver = func() map[string]bool { return map[string]bool{found: true} }
+	HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} }
 	t.Cleanup(func() { HandedOver = before })
 	if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared,
 		noServices, nil, nil); err != nil {
@@ -64,9 +65,9 @@ func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) {
 	if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil {
 		t.Fatal(err)
 	}
-	before := dockerConfig
-	dockerConfig = cfg
-	t.Cleanup(func() { dockerConfig = before })
+	before := runtimeConfigs
+	runtimeConfigs = runtimeFiles{daemonJSON: cfg}
+	t.Cleanup(func() { runtimeConfigs = before })
 	for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
 		for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x",
 			"/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} {
@@ -86,3 +87,52 @@ func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) {
 		t.Errorf("a path beside the data root: %v", err)
 	}
 }
+
+// When a directory was found is kept from the first apply that found it (novox/hq issue 339): the controller
+// tells a directory a send found from one found before it by that time, so it must not move with every apply.
+func TestWhenADirectoryWasFoundIsKept(t *testing.T) {
+	dir := t.TempDir()
+	found := filepath.Join(dir, "found")
+	if err := os.Mkdir(found, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}`
+	_, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
+		noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	first, _ := state.Find("notes.data")
+	if first.FoundAt.IsZero() {
+		t.Fatal("no time the directory was found")
+	}
+	time.Sleep(10 * time.Millisecond)
+	_, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again, _ := state.Find("notes.data"); !again.FoundAt.Equal(first.FoundAt) {
+		t.Fatalf("found at %v, then %v: the time moved with the apply", first.FoundAt, again.FoundAt)
+	}
+}
+
+// A hand-over is for the resource that was used as found there: another declaration that later points another
+// resource at the same path is not handed anything (novox/hq issue 339).
+func TestAHandOverIsForTheResourceThatAskedForIt(t *testing.T) {
+	dir := t.TempDir()
+	found := filepath.Join(dir, "found")
+	if err := os.Mkdir(found, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	before := HandedOver
+	HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} }
+	t.Cleanup(func() { HandedOver = before })
+	other := `{"id":"other.data","type":"directory","path":"` + found + `","mode":"0755"}`
+	if _, _, err := Apply(context.Background(), archHost(t), declare(t, other), store.State{}, store.OriginDeclared,
+		noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if info, _ := os.Stat(found); info.Mode().Perm() != 0o700 {
+		t.Fatalf("a hand-over for notes.data gave other.data the directory: mode %o", info.Mode().Perm())
+	}
+}

internal/apply/placement_guard.go

@@ -14,7 +14,8 @@ package apply
 //     /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc
 //     or /var/lib, never the root itself; owning one is owning everything in it.
 //  2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data
-//     (/var/lib/docker, /var/lib/containers, and the runtime's configured data-root), nor in the node-engine's
+//     (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves
+//     them: runtimeDataRoots), nor in the node-engine's
 //     own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from
 //     those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing
 //     systemPath names.
@@ -41,6 +42,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -58,41 +60,85 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l
 // home. engineTrees are the node-engine's own, which only its own module places in.
 var (
 	forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
-		"/var/lib/containers"}
+		"/var/lib/containers", "/var/lib/containerd"}
 	// forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is
 	// the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing.
-	forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers"}
-	engineTrees         = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
+	forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers",
+		"/var/lib/containerd"}
+	engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
 	// rootsOnly are trees a directory below is root's, and an access is never in.
 	rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"}
 )
 
-// dockerConfig is the container runtime's configuration, whose data-root says where it keeps every container's
-// filesystem. A variable so a test names its own.
-var dockerConfig = "/etc/docker/daemon.json"
+// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its
+// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph), and podman's
+// storage.conf (graphroot). containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a
+// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the
+// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its
+// account. A variable so a test names its own.
+type runtimeFiles struct {
+	daemonJSON  string
+	units       []string
+	dropInDirs  []string
+	storageConf string
+}
 
-// dockerDataRoot is where the container runtime keeps its data, when its configuration moves it from
-// /var/lib/docker (which forbiddenBelow names already); "" otherwise.
-func dockerDataRoot() string {
-	raw, err := os.ReadFile(dockerConfig)
-	if err != nil {
-		return ""
+var runtimeConfigs = runtimeFiles{
+	daemonJSON:  "/etc/docker/daemon.json",
+	units:       []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"},
+	dropInDirs:  []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"},
+	storageConf: "/etc/containers/storage.conf",
+}
+
+var (
+	dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)("?)(/[^\s"]+)`)
+	graphRoot    = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*"(/[^"]+)"`)
+)
+
+// runtimeDataRoots is every place the container runtimes are configured to keep their data, beyond the default
+// trees forbiddenBelow names: every container's filesystem is there.
+func runtimeDataRoots() []string {
+	var out []string
+	add := func(p string) {
+		if filepath.IsAbs(p) {
+			out = append(out, filepath.Clean(p))
+		}
 	}
-	var c struct {
-		DataRoot string `json:"data-root"`
-		Graph    string `json:"graph"`
+	if raw, err := os.ReadFile(runtimeConfigs.daemonJSON); err == nil {
+		var c struct {
+			DataRoot string `json:"data-root"`
+			Graph    string `json:"graph"`
+		}
+		if json.Unmarshal(raw, &c) == nil {
+			add(c.DataRoot)
+			add(c.Graph)
+		}
 	}
-	if json.Unmarshal(raw, &c) != nil {
-		return ""
+	units := append([]string(nil), runtimeConfigs.units...)
+	for _, dir := range runtimeConfigs.dropInDirs {
+		matches, _ := filepath.Glob(filepath.Join(dir, "*.conf"))
+		units = append(units, matches...)
 	}
-	root := c.DataRoot
-	if root == "" {
-		root = c.Graph
+	for _, u := range units {
+		raw, err := os.ReadFile(u)
+		if err != nil {
+			continue
+		}
+		for _, line := range strings.Split(string(raw), "\n") {
+			if !strings.HasPrefix(strings.TrimSpace(line), "ExecStart") {
+				continue
+			}
+			for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) {
+				add(m[2])
+			}
+		}
 	}
-	if !filepath.IsAbs(root) {
-		return ""
+	if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil {
+		for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) {
+			add(m[1])
+		}
 	}
-	return filepath.Clean(root)
+	return out
 }
 
 // engineModule is the module whose resources may place in the engine's own trees.
@@ -233,10 +279,8 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
 		// (systemPath) — and what a setting can mount at all is a directory or an access, refused above it.
 		trees = append([]string(nil), forbiddenBelowMount...)
 	}
-	// The container runtime's data root, wherever the machine keeps it: every container's filesystem is there.
-	if root := dockerDataRoot(); root != "" {
-		trees = append(trees, root)
-	}
+	// The container runtimes' data, wherever the machine keeps it: every container's filesystem is there.
+	trees = append(trees, runtimeDataRoots()...)
 	for _, tree := range trees {
 		if atOrBelow(path, tree) {
 			return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}

internal/apply/runtime_roots_test.go

@@ -0,0 +1,44 @@
+package apply
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// The container runtimes keep every container's filesystem somewhere the machine may have moved: dockerd's
+// --data-root on its unit's command line or a drop-in, podman's graphroot, containerd's own tree (novox/hq issue
+// 339). Each is refused as a place, an access or a mount, wherever it is said.
+func TestEveryRuntimesDataRootIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	write := func(name, content string) string {
+		t.Helper()
+		p := filepath.Join(dir, name)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		return p
+	}
+	unit := write("lib/docker.service", "[Service]\nExecStart=/usr/bin/dockerd -H fd:// --data-root /data/docker-a\n")
+	dropIn := write("etc/docker.service.d/root.conf", "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --data-root=/data/docker-b --log-level warn\n")
+	storage := write("containers/storage.conf", "[storage]\ndriver = \"overlay\"\n  graphroot = \"/data/podman\"\nrunroot = \"/run/containers/storage\"\n")
+	before := runtimeConfigs
+	runtimeConfigs = runtimeFiles{daemonJSON: filepath.Join(dir, "none.json"), units: []string{unit},
+		dropInDirs: []string{filepath.Dir(dropIn)}, storageConf: storage}
+	t.Cleanup(func() { runtimeConfigs = before })
+
+	for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
+		for _, path := range []string{"/var/lib/containerd", "/var/lib/containerd/io.containerd.content.v1",
+			"/data/docker-a/volumes", "/data/docker-b", "/data/podman/overlay"} {
+			if err := refusePath(path, kind, "notes", ""); err == nil {
+				t.Errorf("%v at %s was taken", kind, path)
+			}
+		}
+		if err := refusePath("/data/media", kind, "notes", ""); err != nil {
+			t.Errorf("%v at /data/media: %v", kind, err)
+		}
+	}
+}

internal/store/handed_over.go

@@ -24,6 +24,9 @@ func HandedOverPath(statePath string) string {
 
 // HandOver is one directory a person handed to the mesh.
 type HandOver struct {
+	// ID is the resource that was used as found there: the hand-over is for it alone, and lapses if another
+	// resource later points at the path.
+	ID   string    `json:"id"`
 	Path string    `json:"path"`
 	At   time.Time `json:"at"`
 	// By is who ran the command, as the machine named them.
@@ -53,7 +56,7 @@ func AddHandedOver(statePath string, h HandOver) error {
 		return err
 	}
 	for _, had := range all {
-		if had.Path == h.Path {
+		if had.Path == h.Path && had.ID == h.ID {
 			return nil
 		}
 	}

internal/store/store.go

@@ -137,6 +137,9 @@ type Applied struct {
 	AsFound bool `json:"as_found,omitempty"`
 	// Declared is, for a directory used as found, the owner and mode that were declared and not given it.
 	Declared string `json:"declared,omitempty"`
+	// FoundAt is, for a directory used as found, when an apply first found it so: kept across applies, so the
+	// controller can tell a directory a send found from one found before it.
+	FoundAt time.Time `json:"found_at,omitempty"`
 
 	// Reads is, for a container, the digest of each file it was created reading — its env-files
 	// and the files mounted into it — by path (novox/hq 04-ISSUES/103).