3 Commits
Author SHA1 Message Date
mesh-admin b232db4e17 Merge pull request 'Issue 339 follow-ups: a directory used as found is said and can be handed over; the runtime's data and any .ssh refused' (#64) from fix/339-found-directories-are-said into main 2026-10-09 00:15:17 +00:00
jochen ae1293e73e Read every runtime's data root, keep when a directory was found, and key a hand-over to its resource
mesh/merge-gate pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery covered: a later merge that contains it was delivered: novox/mesh-host@7c046a8a1d84 (merged as d28ca959 into main, walk plan-17915049347305…
The review of #64 asked for containerd's tree, dockerd's --data-root from its
unit and drop-ins, and podman's graphroot to be refused like docker's own
(hq issue 339); and the controller now tells a directory a send found from
one found before it, which needs the time it was first found kept across
applies instead of the time of the last apply. A hand-over now names the
resource it was for, so it lapses if another declaration points at the path.
2026-10-09 00:55:21 +02:00
jochen 07164e40f3 Say every directory used as found, let a person hand it over, and refuse the runtime's data and any .ssh
mesh/merge-gate pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
A directory the engine used as found kept its owner and mode in silence, so
a module that cannot write it would just fail (hq issue 339 review). Each
one is now a WARNING in every apply's log and its module's resource of kind
directory, stated unhealthy on every look with its owner and what was
declared, which the controller raises as the module's condition; `mesh-host
hand-over <dir>` at the machine hands it to the mesh, and the next apply
gives it the declared owner and mode. The guard also refuses the spool, /opt,
the container runtimes' data (its configured data-root too), and an account's
.ssh as an access, a mount, or a directory anywhere but its own home.
2026-10-09 00:28:01 +02:00
12 changed files with 704 additions and 16 deletions
+149
View File
@@ -0,0 +1,149 @@
package main
import (
"errors"
"fmt"
"os"
"os/user"
"path/filepath"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// A directory used as found is never quiet (novox/hq issue 339).
//
// The engine leaves the owner and mode of a directory it did not make: changing them would be root handing a
// directory it never made to whichever account a declaration named. That is safe and, said nowhere, a module
// whose data directory it cannot write would simply fail. So each such directory is said three ways: a WARNING
// line in every apply's log; a resource of its module of kind `directory`, stated unhealthy on every look,
// which the controller raises as that module's condition naming the directory and its owner; and a way out at
// the machine's terminal, `mesh-host hand-over <path>`, after which the next apply gives it the declared owner
// and mode like any directory of the mesh's.
// foundState is where the node's state is, for the looks to read the directories used as found from; empty
// until the engine runs.
var foundState atomic.Value
// handedOverFrom has the apply read the directories handed over from beside the state at statePath.
func handedOverFrom(statePath string) {
apply.HandedOver = func() map[string]bool {
all, err := store.LoadHandedOver(statePath)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-host: %v; no directory counts as handed over until it reads\n", err)
return nil
}
out := map[string]bool{}
for _, h := range all {
out[apply.HandOverKey(h.ID, h.Path)] = true
}
return out
}
}
// foundDirectories is every directory the node's state records as used as found, sorted by id.
func foundDirectories(known store.State) []store.Applied {
var out []store.Applied
for _, r := range known.Resources {
if r.Type == "directory" && r.AsFound {
out = append(out, r)
}
}
sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
return out
}
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and how to hand
// it over. The path is the target, which the controller keeps as evidence.
func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health {
for _, r := range found {
at := strings.LastIndex(r.ID, ".")
if at <= 0 {
continue
}
owner, mode := "unknown", "unknown"
if info, err := os.Lstat(r.Target); err == nil {
mode = fmt.Sprintf("%o", info.Mode().Perm())
if uid, gid, ok := apply.OwnerOf(info); ok {
owner = fmt.Sprintf("%d:%d", uid, gid)
}
}
declared := r.Declared
if declared == "" {
declared = "another owner or mode"
}
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.ID[:at], Resource: r.ID,
Kind: link.KindDirectory, Target: r.Target, State: link.StateUnhealthy,
Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — "+
"`mesh-host hand-over` at the machine hands it to the mesh", link.ReasonUsedAsFound, owner, mode,
declared),
Since: foundSince(r).UTC()})
}
return h
}
// foundNow is the directories used as found, read from the node's state; none when it cannot be read, which
// the apply says on its own.
func foundNow() []store.Applied {
path, _ := foundState.Load().(string)
if path == "" {
return nil
}
known, err := store.Load(path)
if err != nil {
return nil
}
return foundDirectories(known)
}
// handOverCommand hands one directory used as found to the mesh: run by a person, as root, at the machine.
// It changes nothing on disk itself; the next apply gives the directory its declared owner and mode, and says so.
func handOverCommand(opts options) error {
path := filepath.Clean(opts.file)
if !filepath.IsAbs(path) {
return fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one", opts.file)
}
known, err := store.Load(opts.state)
if err != nil {
return err
}
for _, r := range foundDirectories(known) {
if filepath.Clean(r.Target) != path {
continue
}
by := "a person at the machine"
if u, err := user.Current(); err == nil {
by = u.Username
}
if err := store.AddHandedOver(opts.state, store.HandOver{ID: r.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
return err
}
fmt.Fprintf(opts.out, "%s (%s) is handed to the mesh: the next apply gives it %s, and it is the mesh's from "+
"then on — removed when empty and no longer declared\n", path, r.ID, orUnsaid(r.Declared))
return nil
}
return errors.New(path + " is not a directory this machine uses as found (`mesh-host owned` lists what the " +
"engine applied); nothing was handed over")
}
// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one
// a send found from one found before it. A record from before that was kept says when it was last applied.
func foundSince(r store.Applied) time.Time {
if !r.FoundAt.IsZero() {
return r.FoundAt
}
return r.AppliedAt
}
func orUnsaid(s string) string {
if s == "" {
return "the declared owner and mode"
}
return s
}
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on
// every statement (novox/hq issue 339): the controller raises that as the module's condition.
func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
dir := t.TempDir()
known := store.State{Resources: []store.Applied{
{ID: "notes.data", Type: "directory", Target: dir, AsFound: true, Declared: "root, mode 755", AppliedAt: time.Now()},
{ID: "notes.cache", Type: "directory", Target: dir},
}}
h := withFoundDirectories(&link.Health{}, foundDirectories(known))
if len(h.Resources) != 1 {
t.Fatalf("stated: %+v", h.Resources)
}
r := h.Resources[0]
if r.Module != "notes" || r.Kind != link.KindDirectory || r.State != link.StateUnhealthy || r.Target != dir ||
!strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) || !strings.Contains(r.Reason, "owned by ") ||
!strings.Contains(r.Reason, "root, mode 755") || !strings.Contains(r.Reason, "hand-over") {
t.Fatalf("stated as: %+v", r)
}
}
// `mesh-host hand-over` records a directory used as found, and refuses one that is not.
func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) {
dir := t.TempDir()
statePath := filepath.Join(dir, "state.json")
found := filepath.Join(dir, "found")
if err := store.Save(statePath, store.State{Resources: []store.Applied{
{ID: "notes.data", Type: "directory", Target: found, AsFound: true, Declared: "root, mode 755"},
{ID: "notes.mine", Type: "directory", Target: filepath.Join(dir, "mine")},
}}); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
if err := handOverCommand(options{state: statePath, file: filepath.Join(dir, "mine"), out: &out}); err == nil {
t.Fatal("a directory the mesh made was handed over")
}
if err := handOverCommand(options{state: statePath, file: found + "/", out: &out}); err != nil {
t.Fatal(err)
}
all, err := store.LoadHandedOver(statePath)
if err != nil || len(all) != 1 || all[0].Path != found {
t.Fatalf("recorded: %+v %v", all, err)
}
handedOverFrom(statePath)
if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] {
t.Fatal("the apply does not see the hand-over")
}
if !strings.Contains(out.String(), "root, mode 755") {
t.Fatalf("said: %s", out.String())
}
_ = os.Remove(statePath)
}
+21 -3
View File
@@ -108,6 +108,7 @@ const usage = `mesh-host — the node host
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
owned what this host has applied and still owns
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared owner and mode
version
--json machine-readable output
@@ -203,6 +204,13 @@ func parseArgs(args []string) (string, options, error) {
opts.file = positionals[0]
return command, opts, nil
}
if command == "hand-over" {
if len(positionals) != 1 {
return "", opts, errors.New("hand-over <directory>: the path of one directory used as found")
}
opts.file = positionals[0]
return command, opts, nil
}
if command == "overlay" {
if len(positionals) != 1 {
return "", opts, errors.New("overlay take [--tunnel <iface>]")
@@ -220,7 +228,13 @@ func parseArgs(args []string) (string, options, error) {
func run(ctx context.Context, command string, opts options) error {
jsonOut, timeout := opts.json, opts.timeout
// Every apply this process makes counts what a person handed over, and every look says what is used as
// found (novox/hq issue 339).
handedOverFrom(opts.state)
foundState.Store(opts.state)
switch command {
case "hand-over":
return handOverCommand(opts)
case "key":
return keyCommand(opts)
@@ -1537,9 +1551,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
spaced = sp
}
// Every directory used as found, read from the node's state on every look (novox/hq issue 339).
found := foundNow()
since := time.Since(lastSaid)
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy())
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
continue
}
@@ -1551,7 +1567,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
}
}
if queue.SayHealth(ctx, *withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt)) {
if queue.SayHealth(ctx, *withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
accountSt), found)) {
lastSaid, owed = time.Now(), false
}
}
@@ -1896,7 +1913,8 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
as, _ := a.Look(ctx)
accountSt = &as
}
report.Health = withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt)
report.Health = withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt),
foundDirectories(updated))
}
// Which of this machine's links face outside, for the filter the mesh writes around them
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
+45 -5
View File
@@ -75,8 +75,11 @@ type Outcome struct {
groups []string
// unpacked is, for an archive, what it put on the machine (novox/hq issue 162).
unpacked *store.Unpacked
// asFound is, for a directory, that it is used as it was found (novox/hq issue 339).
asFound bool
// asFound is, for a directory, that it is used as it was found (novox/hq issue 339), and asFoundDeclared
// the owner and mode that were declared and not given it.
asFound bool
asFoundDeclared string
asFoundAt time.Time
// reads is, for a container, the digest of each file it was created reading, by path — so
// the next apply can say which one changed (novox/hq 04-ISSUES/103).
reads map[string]string
@@ -737,6 +740,8 @@ func ApplyMindingWindows(
Groups: outcome.groups,
Unpacked: outcome.unpacked,
AsFound: outcome.asFound,
Declared: outcome.asFoundDeclared,
FoundAt: outcome.asFoundAt,
Holds: holds(resource),
})
if outcome.found != nil {
@@ -769,6 +774,11 @@ func ApplyMindingWindows(
}
}
report.Outcomes = append(report.Outcomes, outcome)
if outcome.asFound {
// Never quiet (novox/hq issue 339): said on every apply, and stated unhealthy on every look until a
// person hands it over (FoundDirectories, `mesh-host hand-over`).
log(fmt.Sprintf(" WARNING %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
}
if handed {
// Recorded in its place, so the record going is forgotten now — and only now: a file whose
// new owner failed or was skipped keeps the old record, and the next apply hands it over.
@@ -1021,6 +1031,11 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
}
if existed && wasBefore {
ours := previous.Target != "" && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.AsFound
// Or handed to the mesh by a person at the machine (`mesh-host hand-over`): then it is given the
// declared owner and mode below, as any directory of the mesh's.
if !ours && HandedOver()[handOverKey(r.ID, r.Path)] {
ours = true
}
if !ours {
owned, err := ownedBy(r.Path, r.Owner)
if err != nil {
@@ -1032,9 +1047,14 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
if owner == "" {
owner = "root"
}
out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner and mode %o are "+
"left, though %s and %o were declared (novox/hq issue 339); set them by hand to hand it to the mesh",
before.Mode().Perm(), owner, mode.Perm())
out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm())
out.asFoundAt = time.Now().UTC()
if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() {
out.asFoundAt = previous.FoundAt
}
out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+
"left, though %s were declared (novox/hq issue 339); `mesh-host hand-over %s` at the machine "+
"hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path)
return out, nil
}
}
@@ -3273,3 +3293,23 @@ func removeFile(a store.Applied, now time.Time) (string, string, error) {
return "kept", fmt.Sprintf("no longer declared, and it held more than the mesh wrote — moved aside to %s; "+
"remove it by hand once you know what it is", aside), nil
}
// HandedOver is every directory a person handed to the mesh at the machine (`mesh-host hand-over`), by
// handOverKey: a found directory there is the mesh's, given its declared owner and mode at the next apply
// (novox/hq issue 339). Set by the engine to read the file beside its state; a test sets its own.
var HandedOver = func() map[string]bool { return nil }
// HandOverKey is one hand-over: the resource it was for and the path. A hand-over is for that resource alone,
// so it lapses if another declaration later points another resource at the same path.
func HandOverKey(id, path string) string { return id + " " + filepath.Clean(path) }
func handOverKey(id, path string) string { return HandOverKey(id, path) }
// ownerText is a directory's owner as numbers, "uid:gid", or "unknown".
func ownerText(info os.FileInfo) string {
uid, gid, ok := ownerOf(info)
if !ok {
return "unknown"
}
return fmt.Sprintf("%d:%d", uid, gid)
}
+138
View File
@@ -0,0 +1,138 @@
package apply
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/store"
)
// A directory used as found is never quiet, and a person at the machine can hand it over (novox/hq issue 339).
func TestAFoundDirectoryIsSaidAndCanBeHandedOver(t *testing.T) {
dir := t.TempDir()
found := filepath.Join(dir, "found")
if err := os.Mkdir(found, 0o700); err != nil {
t.Fatal(err)
}
decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}`
var said []string
log := func(line string) { said = append(said, line) }
_, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
noServices, log, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") {
t.Fatalf("the apply did not warn of a directory used as found: %q", said)
}
if a, _ := state.Find("notes.data"); !a.AsFound || !strings.Contains(a.Declared, "755") {
t.Fatalf("the record does not say what was declared: %+v", a)
}
// Said again at the next apply: a warning once is a warning missed.
said = nil
if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared,
noServices, log, nil); err != nil {
t.Fatal(err)
}
if !strings.Contains(strings.Join(said, "\n"), "WARNING notes.data") {
t.Fatalf("the next apply did not warn again: %q", said)
}
// Handed over: the next apply gives it the declared mode and it is the mesh's.
before := HandedOver
HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} }
t.Cleanup(func() { HandedOver = before })
if _, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared,
noServices, nil, nil); err != nil {
t.Fatal(err)
}
if info, _ := os.Stat(found); info.Mode().Perm() != 0o755 {
t.Fatalf("a directory handed over was not given its declared mode: %o", info.Mode().Perm())
}
if a, _ := state.Find("notes.data"); a.AsFound {
t.Fatal("a directory handed over is still recorded as found")
}
}
// The review's additions to the guard: the spool, /opt, the container runtimes' data wherever the runtime keeps
// it, and an account's .ssh as an access or a mount, or as a directory anywhere but its own home.
func TestTheRuntimesDataAndAnySSHAreRefused(t *testing.T) {
cfg := filepath.Join(t.TempDir(), "daemon.json")
if err := os.WriteFile(cfg, []byte(`{"data-root":"/storage/docker"}`), 0o644); err != nil {
t.Fatal(err)
}
before := runtimeConfigs
runtimeConfigs = runtimeFiles{daemonJSON: cfg}
t.Cleanup(func() { runtimeConfigs = before })
for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
for _, path := range []string{"/var/spool/cron", "/opt/app", "/var/lib/docker/volumes/x",
"/var/lib/containers/storage", "/storage/docker", "/storage/docker/overlay2"} {
if err := refusePath(path, kind, "notes", ""); err == nil {
t.Errorf("%v at %s was taken", kind, path)
}
}
}
for _, path := range []string{"/srv/backup/.ssh", "/srv/backup/.ssh/keys"} {
for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
if err := refusePath(path, kind, "notes", ""); err == nil {
t.Errorf("%v at %s was taken", kind, path)
}
}
}
if err := refusePath("/storage/dockerish", placingAccess, "notes", ""); err != nil {
t.Errorf("a path beside the data root: %v", err)
}
}
// When a directory was found is kept from the first apply that found it (novox/hq issue 339): the controller
// tells a directory a send found from one found before it by that time, so it must not move with every apply.
func TestWhenADirectoryWasFoundIsKept(t *testing.T) {
dir := t.TempDir()
found := filepath.Join(dir, "found")
if err := os.Mkdir(found, 0o700); err != nil {
t.Fatal(err)
}
decl := `{"id":"notes.data","type":"directory","path":"` + found + `","mode":"0755"}`
_, state, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
first, _ := state.Find("notes.data")
if first.FoundAt.IsZero() {
t.Fatal("no time the directory was found")
}
time.Sleep(10 * time.Millisecond)
_, state, err = Apply(context.Background(), archHost(t), declare(t, decl), state, store.OriginDeclared, noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if again, _ := state.Find("notes.data"); !again.FoundAt.Equal(first.FoundAt) {
t.Fatalf("found at %v, then %v: the time moved with the apply", first.FoundAt, again.FoundAt)
}
}
// A hand-over is for the resource that was used as found there: another declaration that later points another
// resource at the same path is not handed anything (novox/hq issue 339).
func TestAHandOverIsForTheResourceThatAskedForIt(t *testing.T) {
dir := t.TempDir()
found := filepath.Join(dir, "found")
if err := os.Mkdir(found, 0o700); err != nil {
t.Fatal(err)
}
before := HandedOver
HandedOver = func() map[string]bool { return map[string]bool{handOverKey("notes.data", found): true} }
t.Cleanup(func() { HandedOver = before })
other := `{"id":"other.data","type":"directory","path":"` + found + `","mode":"0755"}`
if _, _, err := Apply(context.Background(), archHost(t), declare(t, other), store.State{}, store.OriginDeclared,
noServices, nil, nil); err != nil {
t.Fatal(err)
}
if info, _ := os.Stat(found); info.Mode().Perm() != 0o700 {
t.Fatalf("a hand-over for notes.data gave other.data the directory: mode %o", info.Mode().Perm())
}
}
+102 -8
View File
@@ -13,9 +13,14 @@ package apply
// 1. **No directory, access or mount source is one of the machine's own roots, or holds one**: /, /etc,
// /usr, /var, /var/lib, /home, /run and the rest of protectedRoots. Modules place directories BELOW /etc
// or /var/lib, never the root itself; owning one is owning everything in it.
// 2. **Nothing is placed in /proc, /sys, /dev, /boot or /root, nor in the node-engine's own trees** (its
// state, its identity, its installed builds) but by its own module; nothing is mounted from /boot, /root or
// the engine's trees. A mount of a kernel file, a device or the clock is the plumbing systemPath names.
// 2. **Nothing is placed in /proc, /sys, /dev, /boot, /root, /var/spool, /opt or the container runtimes' data
// (/var/lib/docker, /var/lib/containers, /var/lib/containerd, and where the runtimes' configuration moves
// them: runtimeDataRoots), nor in the node-engine's
// own trees** (its state, its identity, its installed builds) but by its own module; nothing is mounted from
// those but /proc, /sys and /dev. A mount of a kernel file, a device or the clock is the plumbing
// systemPath names.
// 2a. **An account's .ssh is never an access or a mount**, and is a directory only below its account's home,
// as that account's (rule 4).
// 3. **A directory below /etc, /usr or /run is root's.** The machine's configuration and programs are read
// as root's word; a directory there owned by another account is that account writing root's word. An
// access is never there at all: the operator's data is not the machine's configuration.
@@ -32,10 +37,12 @@ package apply
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
@@ -52,15 +59,88 @@ var protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/l
// forbiddenBelow are trees nothing is placed in or mounted from: the kernel's, the boot loader's and root's
// home. engineTrees are the node-engine's own, which only its own module places in.
var (
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root"}
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
"/var/lib/containers", "/var/lib/containerd"}
// forbiddenBelowMount are the trees no container mounts from: a mount of the kernel's files and devices is
// the plumbing a container may need (systemPath); root's home and the boot loader's are no plumbing.
forbiddenBelowMount = []string{"/boot", "/root"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
forbiddenBelowMount = []string{"/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers",
"/var/lib/containerd"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
// rootsOnly are trees a directory below is root's, and an access is never in.
rootsOnly = []string{"/etc", "/usr", "/run", "/var/run"}
)
// runtimeFiles are where the container runtimes say where they keep their data: dockerd's daemon.json, its
// unit and the unit's drop-ins (an ExecStart with --data-root, or the older -g/--graph), and podman's
// storage.conf (graphroot). containerd keeps its own under /var/lib/containerd, which forbiddenBelow names; a
// containerd configured elsewhere, and podman's rootless stores under each account's home, are not read: the
// first is no runtime this mesh runs, and the second is below a home, which rule 4 already keeps for its
// account. A variable so a test names its own.
type runtimeFiles struct {
daemonJSON string
units []string
dropInDirs []string
storageConf string
}
var runtimeConfigs = runtimeFiles{
daemonJSON: "/etc/docker/daemon.json",
units: []string{"/etc/systemd/system/docker.service", "/usr/lib/systemd/system/docker.service", "/lib/systemd/system/docker.service"},
dropInDirs: []string{"/etc/systemd/system/docker.service.d", "/run/systemd/system/docker.service.d", "/usr/lib/systemd/system/docker.service.d"},
storageConf: "/etc/containers/storage.conf",
}
var (
dataRootFlag = regexp.MustCompile(`(?:--data-root|--graph|-g)(?:=|\s+)("?)(/[^\s"]+)`)
graphRoot = regexp.MustCompile(`(?m)^\s*graphroot\s*=\s*"(/[^"]+)"`)
)
// runtimeDataRoots is every place the container runtimes are configured to keep their data, beyond the default
// trees forbiddenBelow names: every container's filesystem is there.
func runtimeDataRoots() []string {
var out []string
add := func(p string) {
if filepath.IsAbs(p) {
out = append(out, filepath.Clean(p))
}
}
if raw, err := os.ReadFile(runtimeConfigs.daemonJSON); err == nil {
var c struct {
DataRoot string `json:"data-root"`
Graph string `json:"graph"`
}
if json.Unmarshal(raw, &c) == nil {
add(c.DataRoot)
add(c.Graph)
}
}
units := append([]string(nil), runtimeConfigs.units...)
for _, dir := range runtimeConfigs.dropInDirs {
matches, _ := filepath.Glob(filepath.Join(dir, "*.conf"))
units = append(units, matches...)
}
for _, u := range units {
raw, err := os.ReadFile(u)
if err != nil {
continue
}
for _, line := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "ExecStart") {
continue
}
for _, m := range dataRootFlag.FindAllStringSubmatch(line, -1) {
add(m[2])
}
}
}
if raw, err := os.ReadFile(runtimeConfigs.storageConf); err == nil {
for _, m := range graphRoot.FindAllStringSubmatch(string(raw), -1) {
add(m[1])
}
}
return out
}
// engineModule is the module whose resources may place in the engine's own trees.
const engineModule = "mesh-host"
@@ -193,12 +273,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
"and owning or mounting it would be owning or mounting everything in it"}
}
}
trees := forbiddenBelow
trees := append([]string(nil), forbiddenBelow...)
if kind == placingMount {
// A mount is the machine's plumbing as often as a module's data — the clock, a kernel file, /dev/null
// (systemPath) — and what a setting can mount at all is a directory or an access, refused above it.
trees = forbiddenBelowMount
trees = append([]string(nil), forbiddenBelowMount...)
}
// The container runtimes' data, wherever the machine keeps it: every container's filesystem is there.
trees = append(trees, runtimeDataRoots()...)
for _, tree := range trees {
if atOrBelow(path, tree) {
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
@@ -225,6 +307,14 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
"one is declared %s's", tree, owner)}
}
}
ssh := false
for _, part := range strings.Split(path, "/") {
ssh = ssh || part == ".ssh"
}
if ssh && kind != placingDirectory {
return &PlacementRefusedError{Path: path, Why: "it is an account's .ssh, which holds its keys and who may " +
"log in as it, and is never an access or a mount"}
}
if kind != placingDirectory {
return nil
}
@@ -235,6 +325,10 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
deepest = home
}
}
if ssh && deepest == "" {
return &PlacementRefusedError{Path: path, Why: "a .ssh directory is placed only below its account's home, " +
"as that account's"}
}
if deepest != "" {
if a := homes[deepest]; !ownedByAccount(owner, a) {
if owner == "" {
+44
View File
@@ -0,0 +1,44 @@
package apply
import (
"os"
"path/filepath"
"testing"
)
// The container runtimes keep every container's filesystem somewhere the machine may have moved: dockerd's
// --data-root on its unit's command line or a drop-in, podman's graphroot, containerd's own tree (novox/hq issue
// 339). Each is refused as a place, an access or a mount, wherever it is said.
func TestEveryRuntimesDataRootIsRefused(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) string {
t.Helper()
p := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
return p
}
unit := write("lib/docker.service", "[Service]\nExecStart=/usr/bin/dockerd -H fd:// --data-root /data/docker-a\n")
dropIn := write("etc/docker.service.d/root.conf", "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --data-root=/data/docker-b --log-level warn\n")
storage := write("containers/storage.conf", "[storage]\ndriver = \"overlay\"\n graphroot = \"/data/podman\"\nrunroot = \"/run/containers/storage\"\n")
before := runtimeConfigs
runtimeConfigs = runtimeFiles{daemonJSON: filepath.Join(dir, "none.json"), units: []string{unit},
dropInDirs: []string{filepath.Dir(dropIn)}, storageConf: storage}
t.Cleanup(func() { runtimeConfigs = before })
for _, kind := range []placing{placingDirectory, placingAccess, placingMount} {
for _, path := range []string{"/var/lib/containerd", "/var/lib/containerd/io.containerd.content.v1",
"/data/docker-a/volumes", "/data/docker-b", "/data/podman/overlay"} {
if err := refusePath(path, kind, "notes", ""); err == nil {
t.Errorf("%v at %s was taken", kind, path)
}
}
if err := refusePath("/data/media", kind, "notes", ""); err != nil {
t.Errorf("%v at /data/media: %v", kind, err)
}
}
}
+36
View File
@@ -0,0 +1,36 @@
package apply
import (
"fmt"
"os"
"path/filepath"
"testing"
)
// An account's .ssh below its own home, as that account's, is still placed (the ssh-client module's); as an
// access or a mount it is refused there too (novox/hq issue 339).
func TestAnAccountsOwnSSHIsPlacedAndNeverMounted(t *testing.T) {
dir := t.TempDir()
home := filepath.Join(dir, "home", "alice")
passwd := filepath.Join(dir, "passwd")
if err := os.WriteFile(passwd, fmt.Appendf(nil, "alice:x:4242:4242::%s:/bin/sh\n", home), 0o644); err != nil {
t.Fatal(err)
}
before := passwdFile
passwdFile = passwd
t.Cleanup(func() { passwdFile = before })
if err := refusePath(home+"/.ssh", placingDirectory, "ssh-client", "alice"); err != nil {
t.Fatalf("alice's own .ssh: %v", err)
}
if err := refusePath(home+"/.ssh/config.d", placingDirectory, "ssh-client", "alice"); err != nil {
t.Fatalf("alice's own .ssh/config.d: %v", err)
}
for _, kind := range []placing{placingAccess, placingMount} {
if err := refusePath(home+"/.ssh", kind, "notes", ""); err == nil {
t.Errorf("%v of alice's .ssh was taken", kind)
}
}
if err := refusePath(home+"/.ssh", placingDirectory, "notes", "4243"); err == nil {
t.Error("another account's directory at alice's .ssh was taken")
}
}
+3
View File
@@ -456,3 +456,6 @@ func ownAll(root, owner string) error {
func ownerOf(info os.FileInfo) (uid, gid int, ok bool) {
return statOwner(info)
}
// OwnerOf is a file's numeric owner, as the engine reads it.
func OwnerOf(info os.FileInfo) (uid, gid int, ok bool) { return statOwner(info) }
+8
View File
@@ -260,6 +260,14 @@ const KindUnit = "unit"
// reason then starting "relogin needed".
const KindAccount = "account"
// KindDirectory is a directory a module declares that the engine uses as found (novox/hq issue 339): there before
// the mesh, with another owner or mode than declared, and left so. Said unhealthy on every look, its reason
// starting ReasonUsedAsFound, until a person hands it to the mesh at the machine (`mesh-host hand-over`).
const KindDirectory = "directory"
// ReasonUsedAsFound starts the reason of a directory used as found.
const ReasonUsedAsFound = "used as found:"
// UnitsHealth is the machine's service managers in one statement (issue 315).
type UnitsHealth struct {
// State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no
+87
View File
@@ -0,0 +1,87 @@
package store
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"time"
)
// HandedOverName is the file, beside the node's state, of the directories a person handed to the mesh at the
// machine (novox/hq issue 339). Its own file rather than a field of the state: the engine saves its state at
// the end of every apply, from what it loaded at the start, and a hand-over written into the state during an
// apply would be lost to that save.
const HandedOverName = "handed-over.json"
// HandedOverPath is that file beside the state at statePath.
func HandedOverPath(statePath string) string {
return filepath.Join(filepath.Dir(statePath), HandedOverName)
}
// HandOver is one directory a person handed to the mesh.
type HandOver struct {
// ID is the resource that was used as found there: the hand-over is for it alone, and lapses if another
// resource later points at the path.
ID string `json:"id"`
Path string `json:"path"`
At time.Time `json:"at"`
// By is who ran the command, as the machine named them.
By string `json:"by,omitempty"`
}
// LoadHandedOver reads every hand-over. None is an empty list; a file that cannot be read is an error.
func LoadHandedOver(statePath string) ([]HandOver, error) {
raw, err := os.ReadFile(HandedOverPath(statePath))
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("reading the directories handed to the mesh: %w", err)
}
var out []HandOver
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("the directories handed to the mesh (%s) are unreadable: %w", HandedOverPath(statePath), err)
}
return out, nil
}
// AddHandedOver records a hand-over, once per path, atomically.
func AddHandedOver(statePath string, h HandOver) error {
all, err := LoadHandedOver(statePath)
if err != nil {
return err
}
for _, had := range all {
if had.Path == h.Path && had.ID == h.ID {
return nil
}
}
all = append(all, h)
sort.Slice(all, func(a, b int) bool { return all[a].Path < all[b].Path })
raw, err := json.MarshalIndent(all, "", " ")
if err != nil {
return err
}
path := HandedOverPath(statePath)
tmp, err := os.CreateTemp(filepath.Dir(path), ".handed-over-*.json")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if _, err := tmp.Write(append(raw, '\n')); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
+5
View File
@@ -135,6 +135,11 @@ type Applied struct {
// declared owner and mode (novox/hq issue 339): the mesh uses it as found, never changing its owner or
// mode, and never removes it. Absent on a record from before: such a directory is the mesh's.
AsFound bool `json:"as_found,omitempty"`
// Declared is, for a directory used as found, the owner and mode that were declared and not given it.
Declared string `json:"declared,omitempty"`
// FoundAt is, for a directory used as found, when an apply first found it so: kept across applies, so the
// controller can tell a directory a send found from one found before it.
FoundAt time.Time `json:"found_at,omitempty"`
// Reads is, for a container, the digest of each file it was created reading — its env-files
// and the files mounted into it — by path (novox/hq 04-ISSUES/103).