From 07164e40f393381646275c54fce7f3337056adbe Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 00:28:01 +0200 Subject: [PATCH 1/2] Say every directory used as found, let a person hand it over, and refuse the runtime's data and any .ssh A directory the engine used as found kept its owner and mode in silence, so a module that cannot write it would just fail (hq issue 339 review). Each one is now a WARNING in every apply's log and its module's resource of kind directory, stated unhealthy on every look with its owner and what was declared, which the controller raises as the module's condition; `mesh-host hand-over ` at the machine hands it to the mesh, and the next apply gives it the declared owner and mode. The guard also refuses the spool, /opt, the container runtimes' data (its configured data-root too), and an account's .ssh as an access, a mount, or a directory anywhere but its own home. --- cmd/mesh-host/found.go | 140 +++++++++++++++++++++++++++ cmd/mesh-host/found_test.go | 66 +++++++++++++ cmd/mesh-host/main.go | 24 ++++- internal/apply/apply.go | 38 +++++++- internal/apply/found_is_said_test.go | 88 +++++++++++++++++ internal/apply/placement_guard.go | 64 ++++++++++-- internal/apply/ssh_home_test.go | 36 +++++++ internal/apply/user.go | 3 + internal/link/messages.go | 8 ++ internal/store/handed_over.go | 84 ++++++++++++++++ internal/store/store.go | 2 + 11 files changed, 538 insertions(+), 15 deletions(-) create mode 100644 cmd/mesh-host/found.go create mode 100644 cmd/mesh-host/found_test.go create mode 100644 internal/apply/found_is_said_test.go create mode 100644 internal/apply/ssh_home_test.go create mode 100644 internal/store/handed_over.go diff --git a/cmd/mesh-host/found.go b/cmd/mesh-host/found.go new file mode 100644 index 0000000..3912efb --- /dev/null +++ b/cmd/mesh-host/found.go @@ -0,0 +1,140 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/user" + "path/filepath" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// A directory used as found is never quiet (novox/hq issue 339). +// +// The engine leaves the owner and mode of a directory it did not make: changing them would be root handing a +// directory it never made to whichever account a declaration named. That is safe and, said nowhere, a module +// whose data directory it cannot write would simply fail. So each such directory is said three ways: a WARNING +// line in every apply's log; a resource of its module of kind `directory`, stated unhealthy on every look, +// which the controller raises as that module's condition naming the directory and its owner; and a way out at +// the machine's terminal, `mesh-host hand-over `, after which the next apply gives it the declared owner +// and mode like any directory of the mesh's. + +// foundState is where the node's state is, for the looks to read the directories used as found from; empty +// until the engine runs. +var foundState atomic.Value + +// handedOverFrom has the apply read the directories handed over from beside the state at statePath. +func handedOverFrom(statePath string) { + apply.HandedOver = func() map[string]bool { + all, err := store.LoadHandedOver(statePath) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: %v; no directory counts as handed over until it reads\n", err) + return nil + } + out := map[string]bool{} + for _, h := range all { + out[filepath.Clean(h.Path)] = true + } + return out + } +} + +// foundDirectories is every directory the node's state records as used as found, sorted by id. +func foundDirectories(known store.State) []store.Applied { + var out []store.Applied + for _, r := range known.Resources { + if r.Type == "directory" && r.AsFound { + out = append(out, r) + } + } + sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) + return out +} + +// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind +// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and how to hand +// it over. The path is the target, which the controller keeps as evidence. +func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health { + for _, r := range found { + at := strings.LastIndex(r.ID, ".") + if at <= 0 { + continue + } + owner, mode := "unknown", "unknown" + if info, err := os.Lstat(r.Target); err == nil { + mode = fmt.Sprintf("%o", info.Mode().Perm()) + if uid, gid, ok := apply.OwnerOf(info); ok { + owner = fmt.Sprintf("%d:%d", uid, gid) + } + } + declared := r.Declared + if declared == "" { + declared = "another owner or mode" + } + h.Resources = append(h.Resources, link.ResourceHealth{Module: r.ID[:at], Resource: r.ID, + Kind: link.KindDirectory, Target: r.Target, State: link.StateUnhealthy, + Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+ + "`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode, + declared), + Since: r.AppliedAt.UTC()}) + } + return h +} + +// foundNow is the directories used as found, read from the node's state; none when it cannot be read, which +// the apply says on its own. +func foundNow() []store.Applied { + path, _ := foundState.Load().(string) + if path == "" { + return nil + } + known, err := store.Load(path) + if err != nil { + return nil + } + return foundDirectories(known) +} + +// handOverCommand hands one directory used as found to the mesh: run by a person, as root, at the machine. +// It changes nothing on disk itself; the next apply gives the directory its declared owner and mode, and says so. +func handOverCommand(opts options) error { + path := filepath.Clean(opts.file) + if !filepath.IsAbs(path) { + return fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one", opts.file) + } + known, err := store.Load(opts.state) + if err != nil { + return err + } + for _, r := range foundDirectories(known) { + if filepath.Clean(r.Target) != path { + continue + } + by := "a person at the machine" + if u, err := user.Current(); err == nil { + by = u.Username + } + if err := store.AddHandedOver(opts.state, store.HandOver{Path: path, At: time.Now().UTC(), By: by}); err != nil { + return err + } + fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+ + "then on — removed when empty and no longer declared\n", path, r.ID, orUnsaid(r.Declared)) + return nil + } + return errors.New(path + " is not a directory this machine uses as found (`mesh-host owned` lists what the " + + "engine applied); nothing was handed over") +} + +func orUnsaid(s string) string { + if s == "" { + return "the declared owner and mode" + } + return s +} diff --git a/cmd/mesh-host/found_test.go b/cmd/mesh-host/found_test.go new file mode 100644 index 0000000..8e2b70b --- /dev/null +++ b/cmd/mesh-host/found_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on +// every statement (novox/hq issue 339): the controller raises that as the module's condition. +func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) { + dir := t.TempDir() + known := store.State{Resources: []store.Applied{ + {ID: "notes.data", Type: "directory", Target: dir, AsFound: true, Declared: "root, mode 755", AppliedAt: time.Now()}, + {ID: "notes.cache", Type: "directory", Target: dir}, + }} + h := withFoundDirectories(&link.Health{}, foundDirectories(known)) + if len(h.Resources) != 1 { + t.Fatalf("stated: %+v", h.Resources) + } + r := h.Resources[0] + if r.Module != "notes" || r.Kind != link.KindDirectory || r.State != link.StateUnhealthy || r.Target != dir || + !strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) || !strings.Contains(r.Reason, "owned by ") || + !strings.Contains(r.Reason, "root, mode 755") || !strings.Contains(r.Reason, "hand-over") { + t.Fatalf("stated as: %+v", r) + } +} + +// `mesh-host hand-over` records a directory used as found, and refuses one that is not. +func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) { + dir := t.TempDir() + statePath := filepath.Join(dir, "state.json") + found := filepath.Join(dir, "found") + if err := store.Save(statePath, store.State{Resources: []store.Applied{ + {ID: "notes.data", Type: "directory", Target: found, AsFound: true, Declared: "root, mode 755"}, + {ID: "notes.mine", Type: "directory", Target: filepath.Join(dir, "mine")}, + }}); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := handOverCommand(options{state: statePath, file: filepath.Join(dir, "mine"), out: &out}); err == nil { + t.Fatal("a directory the mesh made was handed over") + } + if err := handOverCommand(options{state: statePath, file: found + "/", out: &out}); err != nil { + t.Fatal(err) + } + all, err := store.LoadHandedOver(statePath) + if err != nil || len(all) != 1 || all[0].Path != found { + t.Fatalf("recorded: %+v %v", all, err) + } + handedOverFrom(statePath) + if !apply.HandedOver()[found] { + t.Fatal("the apply does not see the hand-over") + } + if !strings.Contains(out.String(), "root, mode 755") { + t.Fatalf("said: %s", out.String()) + } + _ = os.Remove(statePath) +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 31aa626..3cda687 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -108,6 +108,7 @@ const usage = `mesh-host — the node host overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this node's overlay key and the mesh is told, signed; --tunnel when several are up owned what this host has applied and still owns + hand-over hand a directory used as found to the mesh: the next apply gives it its declared owner and mode version --json machine-readable output @@ -203,6 +204,13 @@ func parseArgs(args []string) (string, options, error) { opts.file = positionals[0] return command, opts, nil } + if command == "hand-over" { + if len(positionals) != 1 { + return "", opts, errors.New("hand-over : the path of one directory used as found") + } + opts.file = positionals[0] + return command, opts, nil + } if command == "overlay" { if len(positionals) != 1 { return "", opts, errors.New("overlay take [--tunnel ]") @@ -220,7 +228,13 @@ func parseArgs(args []string) (string, options, error) { func run(ctx context.Context, command string, opts options) error { jsonOut, timeout := opts.json, opts.timeout + // Every apply this process makes counts what a person handed over, and every look says what is used as + // found (novox/hq issue 339). + handedOverFrom(opts.state) + foundState.Store(opts.state) switch command { + case "hand-over": + return handOverCommand(opts) case "key": return keyCommand(opts) @@ -1537,9 +1551,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } spaced = sp } + // Every directory used as found, read from the node's state on every look (novox/hq issue 339). + found := foundNow() since := time.Since(lastSaid) healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) && - (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) + (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } @@ -1551,7 +1567,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } } } - if queue.SayHealth(ctx, *withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt)) { + if queue.SayHealth(ctx, *withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), + accountSt), found)) { lastSaid, owed = time.Now(), false } } @@ -1896,7 +1913,8 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto as, _ := a.Look(ctx) accountSt = &as } - report.Health = withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt) + report.Health = withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt), + foundDirectories(updated)) } // Which of this machine's links face outside, for the filter the mesh writes around them // (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it, diff --git a/internal/apply/apply.go b/internal/apply/apply.go index b63bc7b..27e9750 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -75,8 +75,10 @@ type Outcome struct { groups []string // unpacked is, for an archive, what it put on the machine (novox/hq issue 162). unpacked *store.Unpacked - // asFound is, for a directory, that it is used as it was found (novox/hq issue 339). - asFound bool + // asFound is, for a directory, that it is used as it was found (novox/hq issue 339), and asFoundDeclared + // the owner and mode that were declared and not given it. + asFound bool + asFoundDeclared string // reads is, for a container, the digest of each file it was created reading, by path — so // the next apply can say which one changed (novox/hq 04-ISSUES/103). reads map[string]string @@ -737,6 +739,7 @@ func ApplyMindingWindows( Groups: outcome.groups, Unpacked: outcome.unpacked, AsFound: outcome.asFound, + Declared: outcome.asFoundDeclared, Holds: holds(resource), }) if outcome.found != nil { @@ -769,6 +772,11 @@ func ApplyMindingWindows( } } report.Outcomes = append(report.Outcomes, outcome) + if outcome.asFound { + // Never quiet (novox/hq issue 339): said on every apply, and stated unhealthy on every look until a + // person hands it over (FoundDirectories, `mesh-host hand-over`). + log(fmt.Sprintf(" WARNING %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } if handed { // Recorded in its place, so the record going is forgotten now — and only now: a file whose // new owner failed or was skipped keeps the old record, and the next apply hands it over. @@ -1021,6 +1029,11 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore } if existed && wasBefore { ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound + // Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the + // declared owner and mode below, as any directory of the mesh's. + if !ours && HandedOver()[filepath.Clean(r.Path)] { + ours = true + } if !ours { owned, err := ownedBy(r.Path, r.Owner) if err != nil { @@ -1032,9 +1045,10 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore if owner == "" { owner = "root" } - out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner and mode %o are "+ - "left, though %s and %o were declared (novox/hq issue 339); set them by hand to hand it to the mesh", - before.Mode().Perm(), owner, mode.Perm()) + out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm()) + out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+ + "left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+ + "hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path) return out, nil } } @@ -3273,3 +3287,17 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) { return "kept", fmt.Sprintf("no longer declared, and it held more than the mesh wrote — moved aside to %s; "+ "remove it by hand once you know what it is", aside), nil } + +// HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by +// path: a found directory there is the mesh's, given its declared owner and mode at the next apply (novox/hq +// issue 339). Set by the engine to read the file beside its state; a test sets its own. +var HandedOver = func() map[string]bool { return nil } + +// ownerText is a directory's owner as numbers, "uid:gid", or "unknown". +func ownerText(info os.FileInfo) string { + uid, gid, ok := ownerOf(info) + if !ok { + return "unknown" + } + return fmt.Sprintf("%d:%d", uid, gid) +} diff --git a/internal/apply/found_is_said_test.go b/internal/apply/found_is_said_test.go new file mode 100644 index 0000000..d6a55cd --- /dev/null +++ b/internal/apply/found_is_said_test.go @@ -0,0 +1,88 @@ +package apply + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// A directory used as found is never quiet, and a person at the machine can hand it over (novox/hq issue 339). +func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}` + var said []string + log := func(line string) { said = append(said, line) } + _, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared, + noServices, log, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") { + t.Fatalf("the apply did not warn of a directory used as found: %q", said) + } + if a, _ := state.Find("notes.data"); !a.AsFound || !strings.Contains(a.Declared, "755") { + t.Fatalf("the record does not say what was declared: %+v", a) + } + // Said again at the next apply: a warning once is a warning missed. + said = nil + if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, + noServices, log, nil); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") { + t.Fatalf("the next apply did not warn again: %q", said) + } + + // Handed over: the next apply gives it the declared mode and it is the mesh's. + before := HandedOver + HandedOver = func() map[string]bool { return map[string]bool{found: true} } + t.Cleanup(func() { HandedOver = before }) + if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, + noServices, nil, nil); err != nil { + t.Fatal(err) + } + if info, _ := os.Stat(found); info.Mode().Perm() != 0o755 { + t.Fatalf("a directory handed over was not given its declared mode: %o", info.Mode().Perm()) + } + if a, _ := state.Find("notes.data"); a.AsFound { + t.Fatal("a directory handed over is still recorded as found") + } +} + +// The review's additions to the guard: the spool, /opt, the container runtimes' data wherever the runtime keeps +// it, and an account's .ssh as an access or a mount, or as a directory anywhere but its own home. +func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) { + cfg := filepath.Join(t.TempDir(), "daemon.json") + if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil { + t.Fatal(err) + } + before := dockerConfig + dockerConfig = cfg + t.Cleanup(func() { dockerConfig = before }) + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x", + "/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + } + for _, path := range []string{"/srv/backup/.ssh", "/srv/backup/.ssh/keys"} { + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + } + if err := refusePath("/storage/dockerish", placingAccess, "notes", ""); err != nil { + t.Errorf("a path beside the data root: %v", err) + } +} diff --git a/internal/apply/placement_guard.go b/internal/apply/placement_guard.go index e253e26..8a21a8a 100644 --- a/internal/apply/placement_guard.go +++ b/internal/apply/placement_guard.go @@ -13,9 +13,13 @@ package apply // 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc, // /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc // or /var/lib, never the root itself; owning one is owning everything in it. -// 2. **Nothing is placed in /proc, /sys, /dev, /boot or /root, nor in the node-engine's own trees** (its -// state, its identity, its installed builds) but by its own module; nothing is mounted from /boot, /root or -// the engine's trees. A mount of a kernel file, a device or the clock is the plumbing systemPath names. +// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data +// (/var/lib/docker, /var/lib/containers, and the runtime's configured data-root), nor in the node-engine's +// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from +// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing +// systemPath names. +// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home, +// as that account's (rule 4). // 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read // as root's word; a directory there owned by another account is that account writing root's word. An // access is never there at all: the operator's data is not the machine's configuration. @@ -32,6 +36,7 @@ package apply import ( "bufio" + "encoding/json" "errors" "fmt" "os" @@ -52,15 +57,44 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l // forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's // home. engineTrees are the node-engine's own, which only its own module places in. var ( - forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root"} + forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", + "/var/lib/containers"} // forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is // the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing. - forbiddenBelowMount = []string{"/boot", "/root"} + forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers"} engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} // rootsOnly are trees a directory below is root's, and an access is never in. rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"} ) +// dockerConfig is the container runtime's configuration, whose data-root says where it keeps every container's +// filesystem. A variable so a test names its own. +var dockerConfig = "/etc/docker/daemon.json" + +// dockerDataRoot is where the container runtime keeps its data, when its configuration moves it from +// /var/lib/docker (which forbiddenBelow names already); "" otherwise. +func dockerDataRoot() string { + raw, err := os.ReadFile(dockerConfig) + if err != nil { + return "" + } + var c struct { + DataRoot string `json:"data-root"` + Graph string `json:"graph"` + } + if json.Unmarshal(raw, &c) != nil { + return "" + } + root := c.DataRoot + if root == "" { + root = c.Graph + } + if !filepath.IsAbs(root) { + return "" + } + return filepath.Clean(root) +} + // engineModule is the module whose resources may place in the engine's own trees. const engineModule = "mesh-host" @@ -193,11 +227,15 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse "and owning or mounting it would be owning or mounting everything in it"} } } - trees := forbiddenBelow + trees := append([]string(nil), forbiddenBelow...) if kind == placingMount { // A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null // (systemPath) — and what a setting can mount at all is a directory or an access, refused above it. - trees = forbiddenBelowMount + trees = append([]string(nil), forbiddenBelowMount...) + } + // The container runtime's data root, wherever the machine keeps it: every container's filesystem is there. + if root := dockerDataRoot(); root != "" { + trees = append(trees, root) } for _, tree := range trees { if atOrBelow(path, tree) { @@ -225,6 +263,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse "one is declared %s's", tree, owner)} } } + ssh := false + for _, part := range strings.Split(path, "/") { + ssh = ssh || part == ".ssh" + } + if ssh && kind != placingDirectory { + return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " + + "log in as it, and is never an access or a mount"} + } if kind != placingDirectory { return nil } @@ -235,6 +281,10 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse deepest = home } } + if ssh && deepest == "" { + return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " + + "as that account's"} + } if deepest != "" { if a := homes[deepest]; !ownedByAccount(owner, a) { if owner == "" { diff --git a/internal/apply/ssh_home_test.go b/internal/apply/ssh_home_test.go new file mode 100644 index 0000000..6049e79 --- /dev/null +++ b/internal/apply/ssh_home_test.go @@ -0,0 +1,36 @@ +package apply + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// An account's .ssh below its own home, as that account's, is still placed (the ssh-client module's); as an +// access or a mount it is refused there too (novox/hq issue 339). +func TestAnAccountsOwnSSHIsPlacedAndNeverMounted(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home", "alice") + passwd := filepath.Join(dir, "passwd") + if err := os.WriteFile(passwd, fmt.Appendf(nil, "alice:x:4242:4242::%s:/bin/sh\n", home), 0o644); err != nil { + t.Fatal(err) + } + before := passwdFile + passwdFile = passwd + t.Cleanup(func() { passwdFile = before }) + if err := refusePath(home+"/.ssh", placingDirectory, "ssh-client", "alice"); err != nil { + t.Fatalf("alice's own .ssh: %v", err) + } + if err := refusePath(home+"/.ssh/config.d", placingDirectory, "ssh-client", "alice"); err != nil { + t.Fatalf("alice's own .ssh/config.d: %v", err) + } + for _, kind := range []placing{placingAccess, placingMount} { + if err := refusePath(home+"/.ssh", kind, "notes", ""); err == nil { + t.Errorf("%v of alice's .ssh was taken", kind) + } + } + if err := refusePath(home+"/.ssh", placingDirectory, "notes", "4243"); err == nil { + t.Error("another account's directory at alice's .ssh was taken") + } +} diff --git a/internal/apply/user.go b/internal/apply/user.go index 7ef0689..5eed355 100644 --- a/internal/apply/user.go +++ b/internal/apply/user.go @@ -456,3 +456,6 @@ func ownAll(root, owner string) error { func ownerOf(info os.FileInfo) (uid, gid int, ok bool) { return statOwner(info) } + +// OwnerOf is a file's numeric owner, as the engine reads it. +func OwnerOf(info os.FileInfo) (uid, gid int, ok bool) { return statOwner(info) } diff --git a/internal/link/messages.go b/internal/link/messages.go index 54d2e3a..4016691 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -260,6 +260,14 @@ const KindUnit = "unit" // reason then starting "relogin needed". const KindAccount = "account" +// KindDirectory is a directory a module declares that the engine uses as found (novox/hq issue 339): there before +// the mesh, with another owner or mode than declared, and left so. Said unhealthy on every look, its reason +// starting ReasonUsedAsFound, until a person hands it to the mesh at the machine (`mesh-host hand-over`). +const KindDirectory = "directory" + +// ReasonUsedAsFound starts the reason of a directory used as found. +const ReasonUsedAsFound = "used as found:" + // UnitsHealth is the machine's service managers in one statement (issue 315). type UnitsHealth struct { // State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no diff --git a/internal/store/handed_over.go b/internal/store/handed_over.go new file mode 100644 index 0000000..ee66c4e --- /dev/null +++ b/internal/store/handed_over.go @@ -0,0 +1,84 @@ +package store + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "time" +) + +// HandedOverName is the file, beside the node's state, of the directories a person handed to the mesh at the +// machine (novox/hq issue 339). Its own file rather than a field of the state: the engine saves its state at +// the end of every apply, from what it loaded at the start, and a hand-over written into the state during an +// apply would be lost to that save. +const HandedOverName = "handed-over.json" + +// HandedOverPath is that file beside the state at statePath. +func HandedOverPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), HandedOverName) +} + +// HandOver is one directory a person handed to the mesh. +type HandOver struct { + Path string `json:"path"` + At time.Time `json:"at"` + // By is who ran the command, as the machine named them. + By string `json:"by,omitempty"` +} + +// LoadHandedOver reads every hand-over. None is an empty list; a file that cannot be read is an error. +func LoadHandedOver(statePath string) ([]HandOver, error) { + raw, err := os.ReadFile(HandedOverPath(statePath)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading the directories handed to the mesh: %w", err) + } + var out []HandOver + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the directories handed to the mesh (%s) are unreadable: %w", HandedOverPath(statePath), err) + } + return out, nil +} + +// AddHandedOver records a hand-over, once per path, atomically. +func AddHandedOver(statePath string, h HandOver) error { + all, err := LoadHandedOver(statePath) + if err != nil { + return err + } + for _, had := range all { + if had.Path == h.Path { + return nil + } + } + all = append(all, h) + sort.Slice(all, func(a, b int) bool { return all[a].Path < all[b].Path }) + raw, err := json.MarshalIndent(all, "", " ") + if err != nil { + return err + } + path := HandedOverPath(statePath) + tmp, err := os.CreateTemp(filepath.Dir(path), ".handed-over-*.json") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(append(raw, '\n')); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/internal/store/store.go b/internal/store/store.go index a7ca2a6..8afe705 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -135,6 +135,8 @@ type Applied struct { // declared owner and mode (novox/hq issue 339): the mesh uses it as found, never changing its owner or // mode, and never removes it. Absent on a record from before: such a directory is the mesh's. AsFound bool `json:"as_found,omitempty"` + // Declared is, for a directory used as found, the owner and mode that were declared and not given it. + Declared string `json:"declared,omitempty"` // Reads is, for a container, the digest of each file it was created reading — its env-files // and the files mounted into it — by path (novox/hq 04-ISSUES/103). From ae1293e73e713520ee823abf7b5a5e58e057ea27 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 00:55:21 +0200 Subject: [PATCH 2/2] Read every runtime's data root, keep when a directory was found, and key a hand-over to its resource The review of #64 asked for containerd's tree, dockerd's --data-root from its unit and drop-ins, and podman's graphroot to be refused like docker's own (hq issue 339); and the controller now tells a directory a send found from one found before it, which needs the time it was first found kept across applies instead of the time of the last apply. A hand-over now names the resource it was for, so it lapses if another declaration points at the path. --- cmd/mesh-host/found.go | 15 +++- cmd/mesh-host/found_test.go | 2 +- internal/apply/apply.go | 18 ++++- internal/apply/found_is_said_test.go | 58 ++++++++++++++-- internal/apply/placement_guard.go | 100 +++++++++++++++++++-------- internal/apply/runtime_roots_test.go | 44 ++++++++++++ internal/store/handed_over.go | 5 +- internal/store/store.go | 3 + 8 files changed, 205 insertions(+), 40 deletions(-) create mode 100644 internal/apply/runtime_roots_test.go diff --git a/cmd/mesh-host/found.go b/cmd/mesh-host/found.go index 3912efb..62db797 100644 --- a/cmd/mesh-host/found.go +++ b/cmd/mesh-host/found.go @@ -40,7 +40,7 @@ func handedOverFrom(statePath string) { } out := map[string]bool{} for _, h := range all { - out[filepath.Clean(h.Path)] = true + out[apply.HandOverKey(h.ID, h.Path)] = true } return out } @@ -83,7 +83,7 @@ func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health { Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+ "`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode, declared), - Since: r.AppliedAt.UTC()}) + Since: foundSince(r).UTC()}) } return h } @@ -121,7 +121,7 @@ func handOverCommand(opts options) error { if u, err := user.Current(); err == nil { by = u.Username } - if err := store.AddHandedOver(opts.state, store.HandOver{Path: path, At: time.Now().UTC(), By: by}); err != nil { + if err := store.AddHandedOver(opts.state, store.HandOver{ID: r.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { return err } fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+ @@ -132,6 +132,15 @@ func handOverCommand(opts options) error { "engine applied); nothing was handed over") } +// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one +// a send found from one found before it. A record from before that was kept says when it was last applied. +func foundSince(r store.Applied) time.Time { + if !r.FoundAt.IsZero() { + return r.FoundAt + } + return r.AppliedAt +} + func orUnsaid(s string) string { if s == "" { return "the declared owner and mode" diff --git a/cmd/mesh-host/found_test.go b/cmd/mesh-host/found_test.go index 8e2b70b..b88dba4 100644 --- a/cmd/mesh-host/found_test.go +++ b/cmd/mesh-host/found_test.go @@ -56,7 +56,7 @@ func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) { t.Fatalf("recorded: %+v %v", all, err) } handedOverFrom(statePath) - if !apply.HandedOver()[found] { + if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] { t.Fatal("the apply does not see the hand-over") } if !strings.Contains(out.String(), "root, mode 755") { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 27e9750..3266ccb 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -79,6 +79,7 @@ type Outcome struct { // the owner and mode that were declared and not given it. asFound bool asFoundDeclared string + asFoundAt time.Time // reads is, for a container, the digest of each file it was created reading, by path — so // the next apply can say which one changed (novox/hq 04-ISSUES/103). reads map[string]string @@ -740,6 +741,7 @@ func ApplyMindingWindows( Unpacked: outcome.unpacked, AsFound: outcome.asFound, Declared: outcome.asFoundDeclared, + FoundAt: outcome.asFoundAt, Holds: holds(resource), }) if outcome.found != nil { @@ -1031,7 +1033,7 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound // Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the // declared owner and mode below, as any directory of the mesh's. - if !ours && HandedOver()[filepath.Clean(r.Path)] { + if !ours && HandedOver()[handOverKey(r.ID, r.Path)] { ours = true } if !ours { @@ -1046,6 +1048,10 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore owner = "root" } out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm()) + out.asFoundAt = time.Now().UTC() + if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() { + out.asFoundAt = previous.FoundAt + } out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+ "left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+ "hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path) @@ -3289,10 +3295,16 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) { } // HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by -// path: a found directory there is the mesh's, given its declared owner and mode at the next apply (novox/hq -// issue 339). Set by the engine to read the file beside its state; a test sets its own. +// handOverKey: a found directory there is the mesh's, given its declared owner and mode at the next apply +// (novox/hq issue 339). Set by the engine to read the file beside its state; a test sets its own. var HandedOver = func() map[string]bool { return nil } +// HandOverKey is one hand-over: the resource it was for and the path. A hand-over is for that resource alone, +// so it lapses if another declaration later points another resource at the same path. +func HandOverKey(id, path string) string { return id + " " + filepath.Clean(path) } + +func handOverKey(id, path string) string { return HandOverKey(id, path) } + // ownerText is a directory's owner as numbers, "uid:gid", or "unknown". func ownerText(info os.FileInfo) string { uid, gid, ok := ownerOf(info) diff --git a/internal/apply/found_is_said_test.go b/internal/apply/found_is_said_test.go index d6a55cd..3c5a530 100644 --- a/internal/apply/found_is_said_test.go +++ b/internal/apply/found_is_said_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/novox/mesh-host/internal/store" ) @@ -43,7 +44,7 @@ func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) { // Handed over: the next apply gives it the declared mode and it is the mesh's. before := HandedOver - HandedOver = func() map[string]bool { return map[string]bool{found: true} } + HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} } t.Cleanup(func() { HandedOver = before }) if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil); err != nil { @@ -64,9 +65,9 @@ func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) { if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil { t.Fatal(err) } - before := dockerConfig - dockerConfig = cfg - t.Cleanup(func() { dockerConfig = before }) + before := runtimeConfigs + runtimeConfigs = runtimeFiles{daemonJSON: cfg} + t.Cleanup(func() { runtimeConfigs = before }) for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x", "/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} { @@ -86,3 +87,52 @@ func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) { t.Errorf("a path beside the data root: %v", err) } } + +// When a directory was found is kept from the first apply that found it (novox/hq issue 339): the controller +// tells a directory a send found from one found before it by that time, so it must not move with every apply. +func TestWhenADirectoryWasFoundIsKept(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}` + _, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared, + noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + first, _ := state.Find("notes.data") + if first.FoundAt.IsZero() { + t.Fatal("no time the directory was found") + } + time.Sleep(10 * time.Millisecond) + _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if again, _ := state.Find("notes.data"); !again.FoundAt.Equal(first.FoundAt) { + t.Fatalf("found at %v, then %v: the time moved with the apply", first.FoundAt, again.FoundAt) + } +} + +// A hand-over is for the resource that was used as found there: another declaration that later points another +// resource at the same path is not handed anything (novox/hq issue 339). +func TestAHandOverIsForTheResourceThatAskedForIt(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + before := HandedOver + HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} } + t.Cleanup(func() { HandedOver = before }) + other := `{"id":"other.data","type":"directory","path":"` + found + `","mode":"0755"}` + if _, _, err := Apply(context.Background(), archHost(t), declare(t, other), store.State{}, store.OriginDeclared, + noServices, nil, nil); err != nil { + t.Fatal(err) + } + if info, _ := os.Stat(found); info.Mode().Perm() != 0o700 { + t.Fatalf("a hand-over for notes.data gave other.data the directory: mode %o", info.Mode().Perm()) + } +} diff --git a/internal/apply/placement_guard.go b/internal/apply/placement_guard.go index 8a21a8a..2433980 100644 --- a/internal/apply/placement_guard.go +++ b/internal/apply/placement_guard.go @@ -14,7 +14,8 @@ package apply // /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc // or /var/lib, never the root itself; owning one is owning everything in it. // 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data -// (/var/lib/docker, /var/lib/containers, and the runtime's configured data-root), nor in the node-engine's +// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves +// them: runtimeDataRoots), nor in the node-engine's // own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from // those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing // systemPath names. @@ -41,6 +42,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strconv" "strings" @@ -58,41 +60,85 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l // home. engineTrees are the node-engine's own, which only its own module places in. var ( forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", - "/var/lib/containers"} + "/var/lib/containers", "/var/lib/containerd"} // forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is // the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing. - forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers"} - engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} + forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers", + "/var/lib/containerd"} + engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} // rootsOnly are trees a directory below is root's, and an access is never in. rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"} ) -// dockerConfig is the container runtime's configuration, whose data-root says where it keeps every container's -// filesystem. A variable so a test names its own. -var dockerConfig = "/etc/docker/daemon.json" +// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its +// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph), and podman's +// storage.conf (graphroot). containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a +// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the +// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its +// account. A variable so a test names its own. +type runtimeFiles struct { + daemonJSON string + units []string + dropInDirs []string + storageConf string +} -// dockerDataRoot is where the container runtime keeps its data, when its configuration moves it from -// /var/lib/docker (which forbiddenBelow names already); "" otherwise. -func dockerDataRoot() string { - raw, err := os.ReadFile(dockerConfig) - if err != nil { - return "" +var runtimeConfigs = runtimeFiles{ + daemonJSON: "/etc/docker/daemon.json", + units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"}, + dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"}, + storageConf: "/etc/containers/storage.conf", +} + +var ( + dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)("?)(/[^\s"]+)`) + graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*"(/[^"]+)"`) +) + +// runtimeDataRoots is every place the container runtimes are configured to keep their data, beyond the default +// trees forbiddenBelow names: every container's filesystem is there. +func runtimeDataRoots() []string { + var out []string + add := func(p string) { + if filepath.IsAbs(p) { + out = append(out, filepath.Clean(p)) + } } - var c struct { - DataRoot string `json:"data-root"` - Graph string `json:"graph"` + if raw, err := os.ReadFile(runtimeConfigs.daemonJSON); err == nil { + var c struct { + DataRoot string `json:"data-root"` + Graph string `json:"graph"` + } + if json.Unmarshal(raw, &c) == nil { + add(c.DataRoot) + add(c.Graph) + } } - if json.Unmarshal(raw, &c) != nil { - return "" + units := append([]string(nil), runtimeConfigs.units...) + for _, dir := range runtimeConfigs.dropInDirs { + matches, _ := filepath.Glob(filepath.Join(dir, "*.conf")) + units = append(units, matches...) } - root := c.DataRoot - if root == "" { - root = c.Graph + for _, u := range units { + raw, err := os.ReadFile(u) + if err != nil { + continue + } + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "ExecStart") { + continue + } + for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) { + add(m[2]) + } + } } - if !filepath.IsAbs(root) { - return "" + if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil { + for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) { + add(m[1]) + } } - return filepath.Clean(root) + return out } // engineModule is the module whose resources may place in the engine's own trees. @@ -233,10 +279,8 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse // (systemPath) — and what a setting can mount at all is a directory or an access, refused above it. trees = append([]string(nil), forbiddenBelowMount...) } - // The container runtime's data root, wherever the machine keeps it: every container's filesystem is there. - if root := dockerDataRoot(); root != "" { - trees = append(trees, root) - } + // The container runtimes' data, wherever the machine keeps it: every container's filesystem is there. + trees = append(trees, runtimeDataRoots()...) for _, tree := range trees { if atOrBelow(path, tree) { return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree} diff --git a/internal/apply/runtime_roots_test.go b/internal/apply/runtime_roots_test.go new file mode 100644 index 0000000..acfecc2 --- /dev/null +++ b/internal/apply/runtime_roots_test.go @@ -0,0 +1,44 @@ +package apply + +import ( + "os" + "path/filepath" + "testing" +) + +// The container runtimes keep every container's filesystem somewhere the machine may have moved: dockerd's +// --data-root on its unit's command line or a drop-in, podman's graphroot, containerd's own tree (novox/hq issue +// 339). Each is refused as a place, an access or a mount, wherever it is said. +func TestEveryRuntimesDataRootIsRefused(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p + } + unit := write("lib/docker.service", "[Service]\nExecStart=/usr/bin/dockerd -H fd:// --data-root /data/docker-a\n") + dropIn := write("etc/docker.service.d/root.conf", "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --data-root=/data/docker-b --log-level warn\n") + storage := write("containers/storage.conf", "[storage]\ndriver = \"overlay\"\n graphroot = \"/data/podman\"\nrunroot = \"/run/containers/storage\"\n") + before := runtimeConfigs + runtimeConfigs = runtimeFiles{daemonJSON: filepath.Join(dir, "none.json"), units: []string{unit}, + dropInDirs: []string{filepath.Dir(dropIn)}, storageConf: storage} + t.Cleanup(func() { runtimeConfigs = before }) + + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + for _, path := range []string{"/var/lib/containerd", "/var/lib/containerd/io.containerd.content.v1", + "/data/docker-a/volumes", "/data/docker-b", "/data/podman/overlay"} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + if err := refusePath("/data/media", kind, "notes", ""); err != nil { + t.Errorf("%v at /data/media: %v", kind, err) + } + } +} diff --git a/internal/store/handed_over.go b/internal/store/handed_over.go index ee66c4e..94fe957 100644 --- a/internal/store/handed_over.go +++ b/internal/store/handed_over.go @@ -24,6 +24,9 @@ func HandedOverPath(statePath string) string { // HandOver is one directory a person handed to the mesh. type HandOver struct { + // ID is the resource that was used as found there: the hand-over is for it alone, and lapses if another + // resource later points at the path. + ID string `json:"id"` Path string `json:"path"` At time.Time `json:"at"` // By is who ran the command, as the machine named them. @@ -53,7 +56,7 @@ func AddHandedOver(statePath string, h HandOver) error { return err } for _, had := range all { - if had.Path == h.Path { + if had.Path == h.Path && had.ID == h.ID { return nil } } diff --git a/internal/store/store.go b/internal/store/store.go index 8afe705..92d48d5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -137,6 +137,9 @@ type Applied struct { AsFound bool `json:"as_found,omitempty"` // Declared is, for a directory used as found, the owner and mode that were declared and not given it. Declared string `json:"declared,omitempty"` + // FoundAt is, for a directory used as found, when an apply first found it so: kept across applies, so the + // controller can tell a directory a send found from one found before it. + FoundAt time.Time `json:"found_at,omitempty"` // Reads is, for a container, the digest of each file it was created reading — its env-files // and the files mounted into it — by path (novox/hq 04-ISSUES/103).