diff --git a/cmd/mesh-host/found.go b/cmd/mesh-host/found.go index 3912efb..62db797 100644 --- a/cmd/mesh-host/found.go +++ b/cmd/mesh-host/found.go @@ -40,7 +40,7 @@ func handedOverFrom(statePath string) { } out := map[string]bool{} for _, h := range all { - out[filepath.Clean(h.Path)] = true + out[apply.HandOverKey(h.ID, h.Path)] = true } return out } @@ -83,7 +83,7 @@ func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health { Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+ "`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode, declared), - Since: r.AppliedAt.UTC()}) + Since: foundSince(r).UTC()}) } return h } @@ -121,7 +121,7 @@ func handOverCommand(opts options) error { if u, err := user.Current(); err == nil { by = u.Username } - if err := store.AddHandedOver(opts.state, store.HandOver{Path: path, At: time.Now().UTC(), By: by}); err != nil { + if err := store.AddHandedOver(opts.state, store.HandOver{ID: r.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { return err } fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+ @@ -132,6 +132,15 @@ func handOverCommand(opts options) error { "engine applied); nothing was handed over") } +// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one +// a send found from one found before it. A record from before that was kept says when it was last applied. +func foundSince(r store.Applied) time.Time { + if !r.FoundAt.IsZero() { + return r.FoundAt + } + return r.AppliedAt +} + func orUnsaid(s string) string { if s == "" { return "the declared owner and mode" diff --git a/cmd/mesh-host/found_test.go b/cmd/mesh-host/found_test.go index 8e2b70b..b88dba4 100644 --- a/cmd/mesh-host/found_test.go +++ b/cmd/mesh-host/found_test.go @@ -56,7 +56,7 @@ func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) { t.Fatalf("recorded: %+v %v", all, err) } handedOverFrom(statePath) - if !apply.HandedOver()[found] { + if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] { t.Fatal("the apply does not see the hand-over") } if !strings.Contains(out.String(), "root, mode 755") { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 27e9750..3266ccb 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -79,6 +79,7 @@ type Outcome struct { // the owner and mode that were declared and not given it. asFound bool asFoundDeclared string + asFoundAt time.Time // reads is, for a container, the digest of each file it was created reading, by path — so // the next apply can say which one changed (novox/hq 04-ISSUES/103). reads map[string]string @@ -740,6 +741,7 @@ func ApplyMindingWindows( Unpacked: outcome.unpacked, AsFound: outcome.asFound, Declared: outcome.asFoundDeclared, + FoundAt: outcome.asFoundAt, Holds: holds(resource), }) if outcome.found != nil { @@ -1031,7 +1033,7 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound // Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the // declared owner and mode below, as any directory of the mesh's. - if !ours && HandedOver()[filepath.Clean(r.Path)] { + if !ours && HandedOver()[handOverKey(r.ID, r.Path)] { ours = true } if !ours { @@ -1046,6 +1048,10 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore owner = "root" } out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm()) + out.asFoundAt = time.Now().UTC() + if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() { + out.asFoundAt = previous.FoundAt + } out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+ "left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+ "hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path) @@ -3289,10 +3295,16 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) { } // HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by -// path: a found directory there is the mesh's, given its declared owner and mode at the next apply (novox/hq -// issue 339). Set by the engine to read the file beside its state; a test sets its own. +// handOverKey: a found directory there is the mesh's, given its declared owner and mode at the next apply +// (novox/hq issue 339). Set by the engine to read the file beside its state; a test sets its own. var HandedOver = func() map[string]bool { return nil } +// HandOverKey is one hand-over: the resource it was for and the path. A hand-over is for that resource alone, +// so it lapses if another declaration later points another resource at the same path. +func HandOverKey(id, path string) string { return id + " " + filepath.Clean(path) } + +func handOverKey(id, path string) string { return HandOverKey(id, path) } + // ownerText is a directory's owner as numbers, "uid:gid", or "unknown". func ownerText(info os.FileInfo) string { uid, gid, ok := ownerOf(info) diff --git a/internal/apply/found_is_said_test.go b/internal/apply/found_is_said_test.go index d6a55cd..3c5a530 100644 --- a/internal/apply/found_is_said_test.go +++ b/internal/apply/found_is_said_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/novox/mesh-host/internal/store" ) @@ -43,7 +44,7 @@ func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) { // Handed over: the next apply gives it the declared mode and it is the mesh's. before := HandedOver - HandedOver = func() map[string]bool { return map[string]bool{found: true} } + HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} } t.Cleanup(func() { HandedOver = before }) if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil); err != nil { @@ -64,9 +65,9 @@ func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) { if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil { t.Fatal(err) } - before := dockerConfig - dockerConfig = cfg - t.Cleanup(func() { dockerConfig = before }) + before := runtimeConfigs + runtimeConfigs = runtimeFiles{daemonJSON: cfg} + t.Cleanup(func() { runtimeConfigs = before }) for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x", "/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} { @@ -86,3 +87,52 @@ func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) { t.Errorf("a path beside the data root: %v", err) } } + +// When a directory was found is kept from the first apply that found it (novox/hq issue 339): the controller +// tells a directory a send found from one found before it by that time, so it must not move with every apply. +func TestWhenADirectoryWasFoundIsKept(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}` + _, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared, + noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + first, _ := state.Find("notes.data") + if first.FoundAt.IsZero() { + t.Fatal("no time the directory was found") + } + time.Sleep(10 * time.Millisecond) + _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if again, _ := state.Find("notes.data"); !again.FoundAt.Equal(first.FoundAt) { + t.Fatalf("found at %v, then %v: the time moved with the apply", first.FoundAt, again.FoundAt) + } +} + +// A hand-over is for the resource that was used as found there: another declaration that later points another +// resource at the same path is not handed anything (novox/hq issue 339). +func TestAHandOverIsForTheResourceThatAskedForIt(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + before := HandedOver + HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} } + t.Cleanup(func() { HandedOver = before }) + other := `{"id":"other.data","type":"directory","path":"` + found + `","mode":"0755"}` + if _, _, err := Apply(context.Background(), archHost(t), declare(t, other), store.State{}, store.OriginDeclared, + noServices, nil, nil); err != nil { + t.Fatal(err) + } + if info, _ := os.Stat(found); info.Mode().Perm() != 0o700 { + t.Fatalf("a hand-over for notes.data gave other.data the directory: mode %o", info.Mode().Perm()) + } +} diff --git a/internal/apply/placement_guard.go b/internal/apply/placement_guard.go index 8a21a8a..2433980 100644 --- a/internal/apply/placement_guard.go +++ b/internal/apply/placement_guard.go @@ -14,7 +14,8 @@ package apply // /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc // or /var/lib, never the root itself; owning one is owning everything in it. // 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data -// (/var/lib/docker, /var/lib/containers, and the runtime's configured data-root), nor in the node-engine's +// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves +// them: runtimeDataRoots), nor in the node-engine's // own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from // those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing // systemPath names. @@ -41,6 +42,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strconv" "strings" @@ -58,41 +60,85 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l // home. engineTrees are the node-engine's own, which only its own module places in. var ( forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", - "/var/lib/containers"} + "/var/lib/containers", "/var/lib/containerd"} // forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is // the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing. - forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers"} - engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} + forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers", + "/var/lib/containerd"} + engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} // rootsOnly are trees a directory below is root's, and an access is never in. rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"} ) -// dockerConfig is the container runtime's configuration, whose data-root says where it keeps every container's -// filesystem. A variable so a test names its own. -var dockerConfig = "/etc/docker/daemon.json" +// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its +// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph), and podman's +// storage.conf (graphroot). containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a +// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the +// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its +// account. A variable so a test names its own. +type runtimeFiles struct { + daemonJSON string + units []string + dropInDirs []string + storageConf string +} -// dockerDataRoot is where the container runtime keeps its data, when its configuration moves it from -// /var/lib/docker (which forbiddenBelow names already); "" otherwise. -func dockerDataRoot() string { - raw, err := os.ReadFile(dockerConfig) - if err != nil { - return "" +var runtimeConfigs = runtimeFiles{ + daemonJSON: "/etc/docker/daemon.json", + units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"}, + dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"}, + storageConf: "/etc/containers/storage.conf", +} + +var ( + dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)("?)(/[^\s"]+)`) + graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*"(/[^"]+)"`) +) + +// runtimeDataRoots is every place the container runtimes are configured to keep their data, beyond the default +// trees forbiddenBelow names: every container's filesystem is there. +func runtimeDataRoots() []string { + var out []string + add := func(p string) { + if filepath.IsAbs(p) { + out = append(out, filepath.Clean(p)) + } } - var c struct { - DataRoot string `json:"data-root"` - Graph string `json:"graph"` + if raw, err := os.ReadFile(runtimeConfigs.daemonJSON); err == nil { + var c struct { + DataRoot string `json:"data-root"` + Graph string `json:"graph"` + } + if json.Unmarshal(raw, &c) == nil { + add(c.DataRoot) + add(c.Graph) + } } - if json.Unmarshal(raw, &c) != nil { - return "" + units := append([]string(nil), runtimeConfigs.units...) + for _, dir := range runtimeConfigs.dropInDirs { + matches, _ := filepath.Glob(filepath.Join(dir, "*.conf")) + units = append(units, matches...) } - root := c.DataRoot - if root == "" { - root = c.Graph + for _, u := range units { + raw, err := os.ReadFile(u) + if err != nil { + continue + } + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "ExecStart") { + continue + } + for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) { + add(m[2]) + } + } } - if !filepath.IsAbs(root) { - return "" + if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil { + for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) { + add(m[1]) + } } - return filepath.Clean(root) + return out } // engineModule is the module whose resources may place in the engine's own trees. @@ -233,10 +279,8 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse // (systemPath) — and what a setting can mount at all is a directory or an access, refused above it. trees = append([]string(nil), forbiddenBelowMount...) } - // The container runtime's data root, wherever the machine keeps it: every container's filesystem is there. - if root := dockerDataRoot(); root != "" { - trees = append(trees, root) - } + // The container runtimes' data, wherever the machine keeps it: every container's filesystem is there. + trees = append(trees, runtimeDataRoots()...) for _, tree := range trees { if atOrBelow(path, tree) { return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree} diff --git a/internal/apply/runtime_roots_test.go b/internal/apply/runtime_roots_test.go new file mode 100644 index 0000000..acfecc2 --- /dev/null +++ b/internal/apply/runtime_roots_test.go @@ -0,0 +1,44 @@ +package apply + +import ( + "os" + "path/filepath" + "testing" +) + +// The container runtimes keep every container's filesystem somewhere the machine may have moved: dockerd's +// --data-root on its unit's command line or a drop-in, podman's graphroot, containerd's own tree (novox/hq issue +// 339). Each is refused as a place, an access or a mount, wherever it is said. +func TestEveryRuntimesDataRootIsRefused(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p + } + unit := write("lib/docker.service", "[Service]\nExecStart=/usr/bin/dockerd -H fd:// --data-root /data/docker-a\n") + dropIn := write("etc/docker.service.d/root.conf", "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --data-root=/data/docker-b --log-level warn\n") + storage := write("containers/storage.conf", "[storage]\ndriver = \"overlay\"\n graphroot = \"/data/podman\"\nrunroot = \"/run/containers/storage\"\n") + before := runtimeConfigs + runtimeConfigs = runtimeFiles{daemonJSON: filepath.Join(dir, "none.json"), units: []string{unit}, + dropInDirs: []string{filepath.Dir(dropIn)}, storageConf: storage} + t.Cleanup(func() { runtimeConfigs = before }) + + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + for _, path := range []string{"/var/lib/containerd", "/var/lib/containerd/io.containerd.content.v1", + "/data/docker-a/volumes", "/data/docker-b", "/data/podman/overlay"} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + if err := refusePath("/data/media", kind, "notes", ""); err != nil { + t.Errorf("%v at /data/media: %v", kind, err) + } + } +} diff --git a/internal/store/handed_over.go b/internal/store/handed_over.go index ee66c4e..94fe957 100644 --- a/internal/store/handed_over.go +++ b/internal/store/handed_over.go @@ -24,6 +24,9 @@ func HandedOverPath(statePath string) string { // HandOver is one directory a person handed to the mesh. type HandOver struct { + // ID is the resource that was used as found there: the hand-over is for it alone, and lapses if another + // resource later points at the path. + ID string `json:"id"` Path string `json:"path"` At time.Time `json:"at"` // By is who ran the command, as the machine named them. @@ -53,7 +56,7 @@ func AddHandedOver(statePath string, h HandOver) error { return err } for _, had := range all { - if had.Path == h.Path { + if had.Path == h.Path && had.ID == h.ID { return nil } } diff --git a/internal/store/store.go b/internal/store/store.go index 8afe705..92d48d5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -137,6 +137,9 @@ type Applied struct { AsFound bool `json:"as_found,omitempty"` // Declared is, for a directory used as found, the owner and mode that were declared and not given it. Declared string `json:"declared,omitempty"` + // FoundAt is, for a directory used as found, when an apply first found it so: kept across applies, so the + // controller can tell a directory a send found from one found before it. + FoundAt time.Time `json:"found_at,omitempty"` // Reads is, for a container, the digest of each file it was created reading — its env-files // and the files mounted into it — by path (novox/hq 04-ISSUES/103).