Merge pull request 'docker: the module's state is in the mesh's directory, never /var/lib/docker (hq issue 452)' (#217) from fix/452-docker-state-is-its-own into main

modules/docker/README.md

@@ -14,7 +14,7 @@ tools below are the module's own.
 | `socket` | `docker.socket` running, enabled at boot | given back as found when the module goes (ADR 0118) |
 | `prune-service`, `prune-timer` | `/etc/systemd/system/docker-prune.{service,timer}`, written whole | removed with the module |
 | `prune` | `docker-prune.timer` running, enabled at boot; restarted when either file changes | stopped and disabled with the module (the mesh made the unit) |
-| `state`, `log-rotation` | the module's own directory, and in it `log-rotation.conf`: the settings `log-max-size` and `log-max-file` as the step judges them (0600) | removed with the module |
+| `state`, `log-rotation` | the directory the mesh places for the module, `<root>/mesh/docker` (`/var/lib/mesh/docker` on a node with the default root), and in it `log-rotation.conf`: the settings `log-max-size` and `log-max-file` as the step judges them (0600). Never `/var/lib/docker`, which is Docker's own and where the node-engine places nothing (novox/hq issues 339, 452) | removed with the module |
 | `log-rotation-check` | a step, `docker-tools check-log-rotation`, run as root when `log-rotation` changes; it refuses a bad value, and the rest of the module (`daemon` with it) is then skipped | removed with the module |
 | `daemon` | `live-restore`, the mesh's registry in `insecure-registries`, `log-driver` and `log-opts` written into `/etc/docker/daemon.json`, beside other keys | each key given back as found when the module goes; only the member this module added leaves the list |
 | `runtime` | `docker.service` running, enabled at boot; reloaded, never restarted, when `daemon` changes | given back as found (ADR 0118) |

modules/docker/cmd/docker-tools/statedir_test.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// Docker's own directory is never the mesh's (novox/hq issue 452). The node-engine places nothing in or
+// below the container runtimes' data (novox/hq issue 339), so a module directory that resolves there is
+// refused on the machine and fails the whole apply. The merge gate composes a change but does not apply it,
+// so it passed #205's `"place": "."`, which the controller resolves to <root>/<module>: /var/lib/docker.
+
+// runtimeTrees are the trees the node-engine refuses to place in (mesh-host internal/apply/placement_guard.go).
+var runtimeTrees = []string{"/var/lib/docker", "/var/lib/containers", "/var/lib/containerd"}
+
+// defaultDataRoot is the controller's root for placed directories when a node states none
+// (mesh-controller internal/catalogue/dir_into.go).
+const defaultDataRoot = "/var/lib"
+
+// placedDirs resolves each directory resource as the controller's dirsFor does on a node with the default
+// data root: a stated path as written, `"place": "."` as <root>/<module>, `"place": "mesh"` as
+// <root>/mesh/<module>, and neither as <root>/<module>/<id>.
+func placedDirs(t *testing.T, m manifestForLogs) map[string]string {
+	t.Helper()
+	dirs := map[string]string{}
+	for _, r := range m.Resources {
+		if r["type"] != "directory" {
+			continue
+		}
+		id := fmt.Sprint(r["id"])
+		if p, ok := r["path"].(string); ok && p != "" {
+			dirs[id] = strings.TrimRight(p, "/")
+			continue
+		}
+		switch r["place"] {
+		case ".":
+			dirs[id] = defaultDataRoot + "/docker"
+		case "mesh":
+			dirs[id] = defaultDataRoot + "/mesh/docker"
+		default:
+			dirs[id] = defaultDataRoot + "/docker/" + id
+		}
+	}
+	return dirs
+}
+
+func inRuntimeTree(p string) string {
+	for _, tree := range runtimeTrees {
+		if p == tree || strings.HasPrefix(p, tree+"/") {
+			return tree
+		}
+	}
+	return ""
+}
+
+// TestNothingIsPlacedInTheRuntimesData: no directory of the module resolves into Docker's own directory or
+// another runtime's, and no file, process or environment value names a path there through ${dir:…} or as
+// written.
+func TestNothingIsPlacedInTheRuntimesData(t *testing.T) {
+	m := readManifestForLogs(t)
+	dirs := placedDirs(t, m)
+	for id, p := range dirs {
+		if tree := inRuntimeTree(p); tree != "" {
+			t.Errorf("directory %q resolves to %s, inside %s: the container runtime's data, never the mesh's (novox/hq issues 339, 452)", id, p, tree)
+		}
+	}
+	ref := regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`)
+	fill := func(s string) string {
+		return ref.ReplaceAllStringFunc(s, func(r string) string {
+			if p, ok := dirs[ref.FindStringSubmatch(r)[1]]; ok {
+				return p
+			}
+			return r
+		})
+	}
+	for _, r := range m.Resources {
+		id := fmt.Sprint(r["id"])
+		var named []string
+		if p, ok := r["path"].(string); ok {
+			named = append(named, p)
+		}
+		if env, ok := r["env"].(map[string]any); ok {
+			for _, v := range env {
+				if s, ok := v.(string); ok {
+					named = append(named, s)
+				}
+			}
+		}
+		for _, p := range named {
+			if tree := inRuntimeTree(fill(p)); tree != "" {
+				t.Errorf("resource %q names %s, inside %s: the container runtime's data, never the mesh's (novox/hq issues 339, 452)", id, fill(p), tree)
+			}
+		}
+	}
+}
+
+// TestLogRotationIsInTheModulesOwnDirectory: the step reads the file the mesh writes, in a directory the mesh
+// places for the module, not one Docker keeps.
+func TestLogRotationIsInTheModulesOwnDirectory(t *testing.T) {
+	m := readManifestForLogs(t)
+	dirs := placedDirs(t, m)
+	if got, want := dirs["state"], "/var/lib/mesh/docker"; got != want {
+		t.Fatalf("the module's state directory resolves to %q, want %q (novox/hq issue 452)", got, want)
+	}
+	file, _ := resourceByID(m, "log-rotation")
+	step, _ := resourceByID(m, "log-rotation-check")
+	if file == nil || step == nil {
+		t.Fatal("log-rotation and log-rotation-check are declared")
+	}
+	env, _ := step["env"].(map[string]any)
+	if file["path"] != env["DOCKER_LOG_ROTATION"] {
+		t.Errorf("the step reads %v, the mesh writes %v: one file", env["DOCKER_LOG_ROTATION"], file["path"])
+	}
+	if file["path"] != "${dir:state}/log-rotation.conf" {
+		t.Errorf("log-rotation is at %v, want ${dir:state}/log-rotation.conf", file["path"])
+	}
+}

modules/docker/module.json

@@ -78,7 +78,7 @@
       "id": "state",
       "type": "directory",
       "mode": "0700",
-      "place": "."
+      "place": "mesh"
     },
     {
       "id": "log-rotation",