diff --git a/cmd/mesh-host/found.go b/cmd/mesh-host/found.go new file mode 100644 index 0000000..62db797 --- /dev/null +++ b/cmd/mesh-host/found.go @@ -0,0 +1,149 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/user" + "path/filepath" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// A directory used as found is never quiet (novox/hq issue 339). +// +// The engine leaves the owner and mode of a directory it did not make: changing them would be root handing a +// directory it never made to whichever account a declaration named. That is safe and, said nowhere, a module +// whose data directory it cannot write would simply fail. So each such directory is said three ways: a WARNING +// line in every apply's log; a resource of its module of kind `directory`, stated unhealthy on every look, +// which the controller raises as that module's condition naming the directory and its owner; and a way out at +// the machine's terminal, `mesh-host hand-over `, after which the next apply gives it the declared owner +// and mode like any directory of the mesh's. + +// foundState is where the node's state is, for the looks to read the directories used as found from; empty +// until the engine runs. +var foundState atomic.Value + +// handedOverFrom has the apply read the directories handed over from beside the state at statePath. +func handedOverFrom(statePath string) { + apply.HandedOver = func() map[string]bool { + all, err := store.LoadHandedOver(statePath) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: %v; no directory counts as handed over until it reads\n", err) + return nil + } + out := map[string]bool{} + for _, h := range all { + out[apply.HandOverKey(h.ID, h.Path)] = true + } + return out + } +} + +// foundDirectories is every directory the node's state records as used as found, sorted by id. +func foundDirectories(known store.State) []store.Applied { + var out []store.Applied + for _, r := range known.Resources { + if r.Type == "directory" && r.AsFound { + out = append(out, r) + } + } + sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) + return out +} + +// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind +// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and how to hand +// it over. The path is the target, which the controller keeps as evidence. +func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health { + for _, r := range found { + at := strings.LastIndex(r.ID, ".") + if at <= 0 { + continue + } + owner, mode := "unknown", "unknown" + if info, err := os.Lstat(r.Target); err == nil { + mode = fmt.Sprintf("%o", info.Mode().Perm()) + if uid, gid, ok := apply.OwnerOf(info); ok { + owner = fmt.Sprintf("%d:%d", uid, gid) + } + } + declared := r.Declared + if declared == "" { + declared = "another owner or mode" + } + h.Resources = append(h.Resources, link.ResourceHealth{Module: r.ID[:at], Resource: r.ID, + Kind: link.KindDirectory, Target: r.Target, State: link.StateUnhealthy, + Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+ + "`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode, + declared), + Since: foundSince(r).UTC()}) + } + return h +} + +// foundNow is the directories used as found, read from the node's state; none when it cannot be read, which +// the apply says on its own. +func foundNow() []store.Applied { + path, _ := foundState.Load().(string) + if path == "" { + return nil + } + known, err := store.Load(path) + if err != nil { + return nil + } + return foundDirectories(known) +} + +// handOverCommand hands one directory used as found to the mesh: run by a person, as root, at the machine. +// It changes nothing on disk itself; the next apply gives the directory its declared owner and mode, and says so. +func handOverCommand(opts options) error { + path := filepath.Clean(opts.file) + if !filepath.IsAbs(path) { + return fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one", opts.file) + } + known, err := store.Load(opts.state) + if err != nil { + return err + } + for _, r := range foundDirectories(known) { + if filepath.Clean(r.Target) != path { + continue + } + by := "a person at the machine" + if u, err := user.Current(); err == nil { + by = u.Username + } + if err := store.AddHandedOver(opts.state, store.HandOver{ID: r.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { + return err + } + fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+ + "then on — removed when empty and no longer declared\n", path, r.ID, orUnsaid(r.Declared)) + return nil + } + return errors.New(path + " is not a directory this machine uses as found (`mesh-host owned` lists what the " + + "engine applied); nothing was handed over") +} + +// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one +// a send found from one found before it. A record from before that was kept says when it was last applied. +func foundSince(r store.Applied) time.Time { + if !r.FoundAt.IsZero() { + return r.FoundAt + } + return r.AppliedAt +} + +func orUnsaid(s string) string { + if s == "" { + return "the declared owner and mode" + } + return s +} diff --git a/cmd/mesh-host/found_test.go b/cmd/mesh-host/found_test.go new file mode 100644 index 0000000..b88dba4 --- /dev/null +++ b/cmd/mesh-host/found_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on +// every statement (novox/hq issue 339): the controller raises that as the module's condition. +func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) { + dir := t.TempDir() + known := store.State{Resources: []store.Applied{ + {ID: "notes.data", Type: "directory", Target: dir, AsFound: true, Declared: "root, mode 755", AppliedAt: time.Now()}, + {ID: "notes.cache", Type: "directory", Target: dir}, + }} + h := withFoundDirectories(&link.Health{}, foundDirectories(known)) + if len(h.Resources) != 1 { + t.Fatalf("stated: %+v", h.Resources) + } + r := h.Resources[0] + if r.Module != "notes" || r.Kind != link.KindDirectory || r.State != link.StateUnhealthy || r.Target != dir || + !strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) || !strings.Contains(r.Reason, "owned by ") || + !strings.Contains(r.Reason, "root, mode 755") || !strings.Contains(r.Reason, "hand-over") { + t.Fatalf("stated as: %+v", r) + } +} + +// `mesh-host hand-over` records a directory used as found, and refuses one that is not. +func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) { + dir := t.TempDir() + statePath := filepath.Join(dir, "state.json") + found := filepath.Join(dir, "found") + if err := store.Save(statePath, store.State{Resources: []store.Applied{ + {ID: "notes.data", Type: "directory", Target: found, AsFound: true, Declared: "root, mode 755"}, + {ID: "notes.mine", Type: "directory", Target: filepath.Join(dir, "mine")}, + }}); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := handOverCommand(options{state: statePath, file: filepath.Join(dir, "mine"), out: &out}); err == nil { + t.Fatal("a directory the mesh made was handed over") + } + if err := handOverCommand(options{state: statePath, file: found + "/", out: &out}); err != nil { + t.Fatal(err) + } + all, err := store.LoadHandedOver(statePath) + if err != nil || len(all) != 1 || all[0].Path != found { + t.Fatalf("recorded: %+v %v", all, err) + } + handedOverFrom(statePath) + if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] { + t.Fatal("the apply does not see the hand-over") + } + if !strings.Contains(out.String(), "root, mode 755") { + t.Fatalf("said: %s", out.String()) + } + _ = os.Remove(statePath) +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 31aa626..3cda687 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -108,6 +108,7 @@ const usage = `mesh-host — the node host overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this node's overlay key and the mesh is told, signed; --tunnel when several are up owned what this host has applied and still owns + hand-over hand a directory used as found to the mesh: the next apply gives it its declared owner and mode version --json machine-readable output @@ -203,6 +204,13 @@ func parseArgs(args []string) (string, options, error) { opts.file = positionals[0] return command, opts, nil } + if command == "hand-over" { + if len(positionals) != 1 { + return "", opts, errors.New("hand-over : the path of one directory used as found") + } + opts.file = positionals[0] + return command, opts, nil + } if command == "overlay" { if len(positionals) != 1 { return "", opts, errors.New("overlay take [--tunnel ]") @@ -220,7 +228,13 @@ func parseArgs(args []string) (string, options, error) { func run(ctx context.Context, command string, opts options) error { jsonOut, timeout := opts.json, opts.timeout + // Every apply this process makes counts what a person handed over, and every look says what is used as + // found (novox/hq issue 339). + handedOverFrom(opts.state) + foundState.Store(opts.state) switch command { + case "hand-over": + return handOverCommand(opts) case "key": return keyCommand(opts) @@ -1537,9 +1551,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } spaced = sp } + // Every directory used as found, read from the node's state on every look (novox/hq issue 339). + found := foundNow() since := time.Since(lastSaid) healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) && - (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) + (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } @@ -1551,7 +1567,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } } } - if queue.SayHealth(ctx, *withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt)) { + if queue.SayHealth(ctx, *withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), + accountSt), found)) { lastSaid, owed = time.Now(), false } } @@ -1896,7 +1913,8 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto as, _ := a.Look(ctx) accountSt = &as } - report.Health = withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt) + report.Health = withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt), + foundDirectories(updated)) } // Which of this machine's links face outside, for the filter the mesh writes around them // (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it, diff --git a/internal/apply/apply.go b/internal/apply/apply.go index b63bc7b..3266ccb 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -75,8 +75,11 @@ type Outcome struct { groups []string // unpacked is, for an archive, what it put on the machine (novox/hq issue 162). unpacked *store.Unpacked - // asFound is, for a directory, that it is used as it was found (novox/hq issue 339). - asFound bool + // asFound is, for a directory, that it is used as it was found (novox/hq issue 339), and asFoundDeclared + // the owner and mode that were declared and not given it. + asFound bool + asFoundDeclared string + asFoundAt time.Time // reads is, for a container, the digest of each file it was created reading, by path — so // the next apply can say which one changed (novox/hq 04-ISSUES/103). reads map[string]string @@ -737,6 +740,8 @@ func ApplyMindingWindows( Groups: outcome.groups, Unpacked: outcome.unpacked, AsFound: outcome.asFound, + Declared: outcome.asFoundDeclared, + FoundAt: outcome.asFoundAt, Holds: holds(resource), }) if outcome.found != nil { @@ -769,6 +774,11 @@ func ApplyMindingWindows( } } report.Outcomes = append(report.Outcomes, outcome) + if outcome.asFound { + // Never quiet (novox/hq issue 339): said on every apply, and stated unhealthy on every look until a + // person hands it over (FoundDirectories, `mesh-host hand-over`). + log(fmt.Sprintf(" WARNING %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } if handed { // Recorded in its place, so the record going is forgotten now — and only now: a file whose // new owner failed or was skipped keeps the old record, and the next apply hands it over. @@ -1021,6 +1031,11 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore } if existed && wasBefore { ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound + // Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the + // declared owner and mode below, as any directory of the mesh's. + if !ours && HandedOver()[handOverKey(r.ID, r.Path)] { + ours = true + } if !ours { owned, err := ownedBy(r.Path, r.Owner) if err != nil { @@ -1032,9 +1047,14 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore if owner == "" { owner = "root" } - out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner and mode %o are "+ - "left, though %s and %o were declared (novox/hq issue 339); set them by hand to hand it to the mesh", - before.Mode().Perm(), owner, mode.Perm()) + out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm()) + out.asFoundAt = time.Now().UTC() + if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() { + out.asFoundAt = previous.FoundAt + } + out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+ + "left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+ + "hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path) return out, nil } } @@ -3273,3 +3293,23 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) { return "kept", fmt.Sprintf("no longer declared, and it held more than the mesh wrote — moved aside to %s; "+ "remove it by hand once you know what it is", aside), nil } + +// HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by +// handOverKey: a found directory there is the mesh's, given its declared owner and mode at the next apply +// (novox/hq issue 339). Set by the engine to read the file beside its state; a test sets its own. +var HandedOver = func() map[string]bool { return nil } + +// HandOverKey is one hand-over: the resource it was for and the path. A hand-over is for that resource alone, +// so it lapses if another declaration later points another resource at the same path. +func HandOverKey(id, path string) string { return id + " " + filepath.Clean(path) } + +func handOverKey(id, path string) string { return HandOverKey(id, path) } + +// ownerText is a directory's owner as numbers, "uid:gid", or "unknown". +func ownerText(info os.FileInfo) string { + uid, gid, ok := ownerOf(info) + if !ok { + return "unknown" + } + return fmt.Sprintf("%d:%d", uid, gid) +} diff --git a/internal/apply/found_is_said_test.go b/internal/apply/found_is_said_test.go new file mode 100644 index 0000000..3c5a530 --- /dev/null +++ b/internal/apply/found_is_said_test.go @@ -0,0 +1,138 @@ +package apply + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/store" +) + +// A directory used as found is never quiet, and a person at the machine can hand it over (novox/hq issue 339). +func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}` + var said []string + log := func(line string) { said = append(said, line) } + _, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared, + noServices, log, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") { + t.Fatalf("the apply did not warn of a directory used as found: %q", said) + } + if a, _ := state.Find("notes.data"); !a.AsFound || !strings.Contains(a.Declared, "755") { + t.Fatalf("the record does not say what was declared: %+v", a) + } + // Said again at the next apply: a warning once is a warning missed. + said = nil + if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, + noServices, log, nil); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") { + t.Fatalf("the next apply did not warn again: %q", said) + } + + // Handed over: the next apply gives it the declared mode and it is the mesh's. + before := HandedOver + HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} } + t.Cleanup(func() { HandedOver = before }) + if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, + noServices, nil, nil); err != nil { + t.Fatal(err) + } + if info, _ := os.Stat(found); info.Mode().Perm() != 0o755 { + t.Fatalf("a directory handed over was not given its declared mode: %o", info.Mode().Perm()) + } + if a, _ := state.Find("notes.data"); a.AsFound { + t.Fatal("a directory handed over is still recorded as found") + } +} + +// The review's additions to the guard: the spool, /opt, the container runtimes' data wherever the runtime keeps +// it, and an account's .ssh as an access or a mount, or as a directory anywhere but its own home. +func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) { + cfg := filepath.Join(t.TempDir(), "daemon.json") + if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil { + t.Fatal(err) + } + before := runtimeConfigs + runtimeConfigs = runtimeFiles{daemonJSON: cfg} + t.Cleanup(func() { runtimeConfigs = before }) + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x", + "/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + } + for _, path := range []string{"/srv/backup/.ssh", "/srv/backup/.ssh/keys"} { + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + } + if err := refusePath("/storage/dockerish", placingAccess, "notes", ""); err != nil { + t.Errorf("a path beside the data root: %v", err) + } +} + +// When a directory was found is kept from the first apply that found it (novox/hq issue 339): the controller +// tells a directory a send found from one found before it by that time, so it must not move with every apply. +func TestWhenADirectoryWasFoundIsKept(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}` + _, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared, + noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + first, _ := state.Find("notes.data") + if first.FoundAt.IsZero() { + t.Fatal("no time the directory was found") + } + time.Sleep(10 * time.Millisecond) + _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if again, _ := state.Find("notes.data"); !again.FoundAt.Equal(first.FoundAt) { + t.Fatalf("found at %v, then %v: the time moved with the apply", first.FoundAt, again.FoundAt) + } +} + +// A hand-over is for the resource that was used as found there: another declaration that later points another +// resource at the same path is not handed anything (novox/hq issue 339). +func TestAHandOverIsForTheResourceThatAskedForIt(t *testing.T) { + dir := t.TempDir() + found := filepath.Join(dir, "found") + if err := os.Mkdir(found, 0o700); err != nil { + t.Fatal(err) + } + before := HandedOver + HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} } + t.Cleanup(func() { HandedOver = before }) + other := `{"id":"other.data","type":"directory","path":"` + found + `","mode":"0755"}` + if _, _, err := Apply(context.Background(), archHost(t), declare(t, other), store.State{}, store.OriginDeclared, + noServices, nil, nil); err != nil { + t.Fatal(err) + } + if info, _ := os.Stat(found); info.Mode().Perm() != 0o700 { + t.Fatalf("a hand-over for notes.data gave other.data the directory: mode %o", info.Mode().Perm()) + } +} diff --git a/internal/apply/placement_guard.go b/internal/apply/placement_guard.go index e253e26..2433980 100644 --- a/internal/apply/placement_guard.go +++ b/internal/apply/placement_guard.go @@ -13,9 +13,14 @@ package apply // 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc, // /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc // or /var/lib, never the root itself; owning one is owning everything in it. -// 2. **Nothing is placed in /proc, /sys, /dev, /boot or /root, nor in the node-engine's own trees** (its -// state, its identity, its installed builds) but by its own module; nothing is mounted from /boot, /root or -// the engine's trees. A mount of a kernel file, a device or the clock is the plumbing systemPath names. +// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data +// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves +// them: runtimeDataRoots), nor in the node-engine's +// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from +// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing +// systemPath names. +// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home, +// as that account's (rule 4). // 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read // as root's word; a directory there owned by another account is that account writing root's word. An // access is never there at all: the operator's data is not the machine's configuration. @@ -32,10 +37,12 @@ package apply import ( "bufio" + "encoding/json" "errors" "fmt" "os" "path/filepath" + "regexp" "strconv" "strings" @@ -52,15 +59,88 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l // forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's // home. engineTrees are the node-engine's own, which only its own module places in. var ( - forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root"} + forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", + "/var/lib/containers", "/var/lib/containerd"} // forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is // the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing. - forbiddenBelowMount = []string{"/boot", "/root"} - engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} + forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers", + "/var/lib/containerd"} + engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} // rootsOnly are trees a directory below is root's, and an access is never in. rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"} ) +// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its +// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph), and podman's +// storage.conf (graphroot). containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a +// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the +// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its +// account. A variable so a test names its own. +type runtimeFiles struct { + daemonJSON string + units []string + dropInDirs []string + storageConf string +} + +var runtimeConfigs = runtimeFiles{ + daemonJSON: "/etc/docker/daemon.json", + units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"}, + dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"}, + storageConf: "/etc/containers/storage.conf", +} + +var ( + dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)("?)(/[^\s"]+)`) + graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*"(/[^"]+)"`) +) + +// runtimeDataRoots is every place the container runtimes are configured to keep their data, beyond the default +// trees forbiddenBelow names: every container's filesystem is there. +func runtimeDataRoots() []string { + var out []string + add := func(p string) { + if filepath.IsAbs(p) { + out = append(out, filepath.Clean(p)) + } + } + if raw, err := os.ReadFile(runtimeConfigs.daemonJSON); err == nil { + var c struct { + DataRoot string `json:"data-root"` + Graph string `json:"graph"` + } + if json.Unmarshal(raw, &c) == nil { + add(c.DataRoot) + add(c.Graph) + } + } + units := append([]string(nil), runtimeConfigs.units...) + for _, dir := range runtimeConfigs.dropInDirs { + matches, _ := filepath.Glob(filepath.Join(dir, "*.conf")) + units = append(units, matches...) + } + for _, u := range units { + raw, err := os.ReadFile(u) + if err != nil { + continue + } + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "ExecStart") { + continue + } + for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) { + add(m[2]) + } + } + } + if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil { + for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) { + add(m[1]) + } + } + return out +} + // engineModule is the module whose resources may place in the engine's own trees. const engineModule = "mesh-host" @@ -193,12 +273,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse "and owning or mounting it would be owning or mounting everything in it"} } } - trees := forbiddenBelow + trees := append([]string(nil), forbiddenBelow...) if kind == placingMount { // A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null // (systemPath) — and what a setting can mount at all is a directory or an access, refused above it. - trees = forbiddenBelowMount + trees = append([]string(nil), forbiddenBelowMount...) } + // The container runtimes' data, wherever the machine keeps it: every container's filesystem is there. + trees = append(trees, runtimeDataRoots()...) for _, tree := range trees { if atOrBelow(path, tree) { return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree} @@ -225,6 +307,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse "one is declared %s's", tree, owner)} } } + ssh := false + for _, part := range strings.Split(path, "/") { + ssh = ssh || part == ".ssh" + } + if ssh && kind != placingDirectory { + return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " + + "log in as it, and is never an access or a mount"} + } if kind != placingDirectory { return nil } @@ -235,6 +325,10 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse deepest = home } } + if ssh && deepest == "" { + return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " + + "as that account's"} + } if deepest != "" { if a := homes[deepest]; !ownedByAccount(owner, a) { if owner == "" { diff --git a/internal/apply/runtime_roots_test.go b/internal/apply/runtime_roots_test.go new file mode 100644 index 0000000..acfecc2 --- /dev/null +++ b/internal/apply/runtime_roots_test.go @@ -0,0 +1,44 @@ +package apply + +import ( + "os" + "path/filepath" + "testing" +) + +// The container runtimes keep every container's filesystem somewhere the machine may have moved: dockerd's +// --data-root on its unit's command line or a drop-in, podman's graphroot, containerd's own tree (novox/hq issue +// 339). Each is refused as a place, an access or a mount, wherever it is said. +func TestEveryRuntimesDataRootIsRefused(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p + } + unit := write("lib/docker.service", "[Service]\nExecStart=/usr/bin/dockerd -H fd:// --data-root /data/docker-a\n") + dropIn := write("etc/docker.service.d/root.conf", "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --data-root=/data/docker-b --log-level warn\n") + storage := write("containers/storage.conf", "[storage]\ndriver = \"overlay\"\n graphroot = \"/data/podman\"\nrunroot = \"/run/containers/storage\"\n") + before := runtimeConfigs + runtimeConfigs = runtimeFiles{daemonJSON: filepath.Join(dir, "none.json"), units: []string{unit}, + dropInDirs: []string{filepath.Dir(dropIn)}, storageConf: storage} + t.Cleanup(func() { runtimeConfigs = before }) + + for _, kind := range []placing{placingDirectory, placingAccess, placingMount} { + for _, path := range []string{"/var/lib/containerd", "/var/lib/containerd/io.containerd.content.v1", + "/data/docker-a/volumes", "/data/docker-b", "/data/podman/overlay"} { + if err := refusePath(path, kind, "notes", ""); err == nil { + t.Errorf("%v at %s was taken", kind, path) + } + } + if err := refusePath("/data/media", kind, "notes", ""); err != nil { + t.Errorf("%v at /data/media: %v", kind, err) + } + } +} diff --git a/internal/apply/ssh_home_test.go b/internal/apply/ssh_home_test.go new file mode 100644 index 0000000..6049e79 --- /dev/null +++ b/internal/apply/ssh_home_test.go @@ -0,0 +1,36 @@ +package apply + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// An account's .ssh below its own home, as that account's, is still placed (the ssh-client module's); as an +// access or a mount it is refused there too (novox/hq issue 339). +func TestAnAccountsOwnSSHIsPlacedAndNeverMounted(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home", "alice") + passwd := filepath.Join(dir, "passwd") + if err := os.WriteFile(passwd, fmt.Appendf(nil, "alice:x:4242:4242::%s:/bin/sh\n", home), 0o644); err != nil { + t.Fatal(err) + } + before := passwdFile + passwdFile = passwd + t.Cleanup(func() { passwdFile = before }) + if err := refusePath(home+"/.ssh", placingDirectory, "ssh-client", "alice"); err != nil { + t.Fatalf("alice's own .ssh: %v", err) + } + if err := refusePath(home+"/.ssh/config.d", placingDirectory, "ssh-client", "alice"); err != nil { + t.Fatalf("alice's own .ssh/config.d: %v", err) + } + for _, kind := range []placing{placingAccess, placingMount} { + if err := refusePath(home+"/.ssh", kind, "notes", ""); err == nil { + t.Errorf("%v of alice's .ssh was taken", kind) + } + } + if err := refusePath(home+"/.ssh", placingDirectory, "notes", "4243"); err == nil { + t.Error("another account's directory at alice's .ssh was taken") + } +} diff --git a/internal/apply/user.go b/internal/apply/user.go index 7ef0689..5eed355 100644 --- a/internal/apply/user.go +++ b/internal/apply/user.go @@ -456,3 +456,6 @@ func ownAll(root, owner string) error { func ownerOf(info os.FileInfo) (uid, gid int, ok bool) { return statOwner(info) } + +// OwnerOf is a file's numeric owner, as the engine reads it. +func OwnerOf(info os.FileInfo) (uid, gid int, ok bool) { return statOwner(info) } diff --git a/internal/link/messages.go b/internal/link/messages.go index 54d2e3a..4016691 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -260,6 +260,14 @@ const KindUnit = "unit" // reason then starting "relogin needed". const KindAccount = "account" +// KindDirectory is a directory a module declares that the engine uses as found (novox/hq issue 339): there before +// the mesh, with another owner or mode than declared, and left so. Said unhealthy on every look, its reason +// starting ReasonUsedAsFound, until a person hands it to the mesh at the machine (`mesh-host hand-over`). +const KindDirectory = "directory" + +// ReasonUsedAsFound starts the reason of a directory used as found. +const ReasonUsedAsFound = "used as found:" + // UnitsHealth is the machine's service managers in one statement (issue 315). type UnitsHealth struct { // State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no diff --git a/internal/store/handed_over.go b/internal/store/handed_over.go new file mode 100644 index 0000000..94fe957 --- /dev/null +++ b/internal/store/handed_over.go @@ -0,0 +1,87 @@ +package store + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "time" +) + +// HandedOverName is the file, beside the node's state, of the directories a person handed to the mesh at the +// machine (novox/hq issue 339). Its own file rather than a field of the state: the engine saves its state at +// the end of every apply, from what it loaded at the start, and a hand-over written into the state during an +// apply would be lost to that save. +const HandedOverName = "handed-over.json" + +// HandedOverPath is that file beside the state at statePath. +func HandedOverPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), HandedOverName) +} + +// HandOver is one directory a person handed to the mesh. +type HandOver struct { + // ID is the resource that was used as found there: the hand-over is for it alone, and lapses if another + // resource later points at the path. + ID string `json:"id"` + Path string `json:"path"` + At time.Time `json:"at"` + // By is who ran the command, as the machine named them. + By string `json:"by,omitempty"` +} + +// LoadHandedOver reads every hand-over. None is an empty list; a file that cannot be read is an error. +func LoadHandedOver(statePath string) ([]HandOver, error) { + raw, err := os.ReadFile(HandedOverPath(statePath)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading the directories handed to the mesh: %w", err) + } + var out []HandOver + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the directories handed to the mesh (%s) are unreadable: %w", HandedOverPath(statePath), err) + } + return out, nil +} + +// AddHandedOver records a hand-over, once per path, atomically. +func AddHandedOver(statePath string, h HandOver) error { + all, err := LoadHandedOver(statePath) + if err != nil { + return err + } + for _, had := range all { + if had.Path == h.Path && had.ID == h.ID { + return nil + } + } + all = append(all, h) + sort.Slice(all, func(a, b int) bool { return all[a].Path < all[b].Path }) + raw, err := json.MarshalIndent(all, "", " ") + if err != nil { + return err + } + path := HandedOverPath(statePath) + tmp, err := os.CreateTemp(filepath.Dir(path), ".handed-over-*.json") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(append(raw, '\n')); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/internal/store/store.go b/internal/store/store.go index a7ca2a6..92d48d5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -135,6 +135,11 @@ type Applied struct { // declared owner and mode (novox/hq issue 339): the mesh uses it as found, never changing its owner or // mode, and never removes it. Absent on a record from before: such a directory is the mesh's. AsFound bool `json:"as_found,omitempty"` + // Declared is, for a directory used as found, the owner and mode that were declared and not given it. + Declared string `json:"declared,omitempty"` + // FoundAt is, for a directory used as found, when an apply first found it so: kept across applies, so the + // controller can tell a directory a send found from one found before it. + FoundAt time.Time `json:"found_at,omitempty"` // Reads is, for a container, the digest of each file it was created reading — its env-files // and the files mounted into it — by path (novox/hq 04-ISSUES/103).