diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 8b237c2..9267b9b 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1120,7 +1120,11 @@ func runLink(ctx context.Context, opts options) error { // And which units its service managers say failed, and whose each is (novox/hq issue 315). unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor}) // And whether an account a module put in a group has it where it runs (novox/hq ADR 0252). - accountJudge = accounts.New(accounts.Exec{Run: accounts.CLocale, Cache: &accounts.SetuidCache{Every: time.Hour}}) + // Its search for setuid programs runs to its end at idle priority, and the last complete one is kept + // beside this state, so a restart judges from it (novox/hq issue 361). + setuidSearch = &accounts.SetuidCache{Path: filepath.Join(filepath.Dir(opts.state), accounts.SetuidSearchFile), + Say: say} + accountJudge = accounts.New(accounts.Exec{Run: accounts.CLocale, SearchRun: accounts.Idle, Cache: setuidSearch}) } // **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR @@ -1431,6 +1435,10 @@ var unitJudge *units.Judge // ADR 0252); nil where judging is. var accountJudge *accounts.Judge +// setuidSearch is accountJudge's search for setuid programs, asked to search again after an apply that moved +// what root controls (novox/hq issue 361); nil where accountJudge is. +var setuidSearch *accounts.SetuidCache + // netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a // one-shot command and in a test, which say nothing of the network. var netJudge networkJudge @@ -1891,6 +1899,18 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // either way, and the node is held to it from the next reconcile (novox/hq ADR 0100). updated.Mode = apply.ModeOf(declared) + // Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's + // rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it + // did before it stopped. + moved := outcome.Outcomes + var partial *apply.Error + if errors.As(applyErr, &partial) { + moved = append(append([]apply.Outcome(nil), moved...), partial.Done.Outcomes...) + } + if apply.MovesRoot(moved) { + setuidSearch.SearchAgain() + } + // Saved whichever way it went. Recording only on success would lose the footprint of a // failed apply, and that footprint is on the machine either way. if saveErr := store.Save(opts.state, updated); saveErr != nil { diff --git a/internal/accounts/exec.go b/internal/accounts/exec.go index 67ededc..8260a30 100644 --- a/internal/accounts/exec.go +++ b/internal/accounts/exec.go @@ -34,6 +34,9 @@ type Exec struct { ACL func(path string) ([]ACLEntry, error) // Cache keeps the search for setuid programs between looks; nil searches on every look. Cache *SetuidCache + // SearchRun runs the setuid search's commands (find, and the package manager it asks); nil is Run. A + // serving engine gives Idle, so the walk of every filesystem yields to everything else on the machine. + SearchRun Runner // Now is the clock the cache is kept by; nil is the machine's. Now func() time.Time } diff --git a/internal/accounts/idle_linux.go b/internal/accounts/idle_linux.go new file mode 100644 index 0000000..7ccfa38 --- /dev/null +++ b/internal/accounts/idle_linux.go @@ -0,0 +1,68 @@ +//go:build linux + +package accounts + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "syscall" + + "golang.org/x/sys/unix" +) + +// ioprioIdle is the idle I/O class (IOPRIO_CLASS_IDLE, 3, shifted by IOPRIO_CLASS_SHIFT, 13): the process gets +// the disk only when nothing else asks for it, where the disk's scheduler honours classes (BFQ does). +const ioprioIdle = 3 << 13 + +// Idle runs a command as CLocale does, at the lowest priority the machine has (novox/hq issue 361): CPU nice +// 19 and the idle I/O class, set on the process as soon as it has started — so the search for setuid programs, +// which walks every file and runs as long as that takes, yields to everything else. It dies with the engine +// (the parent-death signal), so a restarted engine never leaves a walk running beside its own. A priority it +// could not lower is a failure: the command is killed rather than walk the disks at full priority. +func Idle(ctx context.Context, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C", "LANGUAGE=C") + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + // The parent-death signal follows the thread that started the child, not the process: held on this + // thread until the child has ended, so the runtime retiring a thread never kills a walk midway. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if err := cmd.Start(); err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + pid := cmd.Process.Pid + if err := lower(pid); err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return "", fmt.Errorf("%s: its priority could not be lowered, so it was stopped: %w", name, err) + } + err := cmd.Wait() + out := stdout.String() + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + return out, fmt.Errorf("%s exited %d: %s", name, exit.ExitCode(), strings.TrimSpace(stderr.String())) + } + return out, fmt.Errorf("%s: %w", name, err) + } + return out, nil +} + +// lower sets a process's CPU nice to 19 and its I/O class to idle. +func lower(pid int) error { + if err := unix.Setpriority(unix.PRIO_PROCESS, pid, 19); err != nil { + return fmt.Errorf("nice: %w", err) + } + if _, _, errno := unix.Syscall(unix.SYS_IOPRIO_SET, 1 /* IOPRIO_WHO_PROCESS */, uintptr(pid), ioprioIdle); errno != 0 { + return fmt.Errorf("I/O class: %w", errno) + } + return nil +} diff --git a/internal/accounts/idle_other.go b/internal/accounts/idle_other.go new file mode 100644 index 0000000..2dff47f --- /dev/null +++ b/internal/accounts/idle_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package accounts + +import "context" + +// Idle is CLocale where the machine offers no idle class this engine sets (novox/hq issue 361). +func Idle(ctx context.Context, name string, args ...string) (string, error) { + return CLocale(ctx, name, args...) +} diff --git a/internal/accounts/setuid_cache.go b/internal/accounts/setuid_cache.go new file mode 100644 index 0000000..b85b648 --- /dev/null +++ b/internal/accounts/setuid_cache.go @@ -0,0 +1,338 @@ +package accounts + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "sync" + "syscall" + "time" + + "github.com/novox/mesh-host/rootsearch" +) + +// SetuidCache keeps the search for setuid programs, which walks every local filesystem, and runs it in the +// background (novox/hq ADR 0266, issue 361): a look never waits for it, and the search itself is never cut +// short. +// +// - **A verdict comes only from a complete search, while it is fresh**: one that walked everything and had the +// package manager answer for every path found, started less than FreshFor ago. Before the first completes, +// or once the last is older, the answer is "not judged yet" (ReasonPending) while a search runs, and "not +// judged" (ReasonIncomplete) after one failed. Never "none found" from a search that did not finish. +// - **It runs to completion**, once at a time, with no bound: a fresh one when the last complete one is Every +// old, when SearchAgain asks (an apply changed what root controls), or when a filesystem appeared that the +// last one did not walk. One that fails is tried again after a back-off that doubles from FailedBackoff up +// to MaxBackoff. +// - **Kept** in Path, beside the engine's other state, with when it started and finished: an engine restart +// serves the last complete search rather than judging nothing until a new walk ends. +// +// Why a complete search stays sound for a day and more is said in the rootsearch package: only root makes a +// setuid-root file, never the agent account. +type SetuidCache struct { + // Every and FreshFor are rootsearch's when zero. + Every, FreshFor time.Duration + // Path is the file the last complete search is kept in; empty keeps it in memory only. The engine's own + // state directory, which only root writes: a file there that is not a regular file root owns and alone + // may write is not read (keptSearch). + Path string + // Say is where a search's end and a kept search refused are said; nil says nothing. + Say func(string) + + mu sync.Mutex + wg sync.WaitGroup + loaded bool + last *setuidResult // the last complete search + running bool + startedAt time.Time // when the search running, or the last one, was started + again bool // asked to search again since the running or last one started + failedAt time.Time + failed error + backoff time.Duration + starts int // how many searches were started, for a test +} + +// setuidResult is one complete search, as kept. +type setuidResult struct { + // Started and Finished are on the caller's clock: Finished is Started plus how long the search took. + Started time.Time `json:"started"` + Finished time.Time `json:"finished"` + // Mounts are the filesystems it walked; Found the setuid-root programs no package owns. + Mounts []string `json:"mounts"` + Found []string `json:"found"` +} + +const ( + // FailedBackoff is the first wait after a search failed, doubling to MaxBackoff. + FailedBackoff = time.Hour + MaxBackoff = 6 * time.Hour + // AgainAfter is the least time between the starts of two searches that SearchAgain or a new filesystem + // asked for: a run of applies does not walk the disks back to back. + AgainAfter = time.Hour + // clockSkew is how far in the future a kept search may say it started and still judge: a clock set back + // further makes it no verdict, never a fresh one. + clockSkew = 5 * time.Minute + // SetuidSearchFile is the file's name beside the engine's state. + SetuidSearchFile = "setuid-search.json" +) + +// SetuidNotJudged is the search's answer when no complete search judges: none has finished yet, the last one is +// older than FreshFor, or every one since failed. The judge says the way is not judged, never that there is none. +type SetuidNotJudged struct { + // Pending is true while a search runs; false after one failed. + Pending bool + Since time.Time + Err error + // Last is when the last complete search started, when there is one too old to judge. + Last time.Time +} + +func (e *SetuidNotJudged) Error() string { + stale := "" + if !e.Last.IsZero() { + stale = fmt.Sprintf("; the last complete one, started %s, is too old to judge", e.Last.Local().Format("2006-01-02 15:04")) + } + if e.Pending { + return fmt.Sprintf("%s: the search for setuid programs, started %s, has not finished yet%s", ReasonPending, + e.Since.Local().Format("2006-01-02 15:04"), stale) + } + return fmt.Sprintf("%s: the search for setuid programs did not complete (last tried %s: %v); it is tried "+ + "again later%s", ReasonIncomplete, e.Since.Local().Format("2006-01-02 15:04"), e.Err, stale) +} + +// The reasons of a root verdict the setuid search could not answer. +const ( + ReasonPending = rootsearch.ReasonPending + ReasonIncomplete = rootsearch.ReasonIncomplete +) + +func (c *SetuidCache) every() time.Duration { + if c.Every > 0 { + return c.Every + } + return rootsearch.Every +} + +func (c *SetuidCache) freshFor() time.Duration { + if c.FreshFor > 0 { + return c.FreshFor + } + return rootsearch.FreshFor +} + +// SearchAgain asks for a new search at the next look: an apply changed what root controls (an account, a group, +// a sudo rule, a package, an action). The last complete search keeps judging until it is too old: only root acts +// through an apply, and the agent account cannot make a setuid-root file. Nil c is nothing to ask. +func (c *SetuidCache) SearchAgain() { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.again = true +} + +// Look answers the setuid-root programs no package owns as the last complete search found them, and when it +// started — while it is fresh — and starts a search in the background when one is due; it never waits for one. +// mounts are the filesystems a search walks now. Nil c searches in place, for a caller that wants to. +// +// **One clock: the caller's.** A search's time is the `now` it was started at, never the wall clock read when it +// finished: due-ness compares a later `now` with it, and mixing the two clocks made a search never due again +// whenever the caller's clock ran behind the wall clock (found while checking issue 356). Measured from the +// start, a result reads at most one search older than it is, which errs towards not judging. +func (c *SetuidCache) Look(now time.Time, mounts []string, + search func(ctx context.Context, mounts []string) ([]string, error)) ([]string, time.Time, error) { + if c == nil { + found, err := search(context.Background(), mounts) + return found, now, err + } + c.mu.Lock() + defer c.mu.Unlock() + if !c.loaded { + c.loaded = true + c.last = c.load() + } + var age time.Duration + if c.last != nil { + age = now.Sub(c.last.Started) + } + fresh := c.last != nil && age > -clockSkew && age < c.freshFor() + asked := c.again || (c.last != nil && !covers(c.last.Mounts, mounts)) + due := !c.running && (c.failed == nil || now.Sub(c.failedAt) >= c.backoff) && + (!fresh || age >= c.every() || (asked && (c.startedAt.IsZero() || now.Sub(c.startedAt) >= AgainAfter))) + if due { + c.start(now, mounts, search) + } + switch { + case fresh: + return slices.Clone(c.last.Found), c.last.Started, nil + case c.running: + return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: c.startedAt, Last: c.lastStarted()} + case c.failed != nil: + return nil, time.Time{}, &SetuidNotJudged{Since: c.failedAt, Err: c.failed, Last: c.lastStarted()} + default: + // Not fresh, not running, never failed: cannot be, as not fresh is due. Never a verdict. + return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: now, Last: c.lastStarted()} + } +} + +func (c *SetuidCache) lastStarted() time.Time { + if c.last == nil { + return time.Time{} + } + return c.last.Started +} + +// start runs one search in the background, with no bound: c.mu held. +func (c *SetuidCache) start(now time.Time, mounts []string, search func(context.Context, []string) ([]string, error)) { + c.running, c.startedAt, c.again = true, now, false + c.starts++ + mounts = slices.Clone(mounts) + began := time.Now() + c.wg.Add(1) + go func() { + defer c.wg.Done() + found, err := search(context.Background(), mounts) + took := time.Since(began) + c.mu.Lock() + defer c.mu.Unlock() + c.running = false + if err != nil { + c.failed, c.failedAt = err, now + c.backoff = min(max(2*c.backoff, FailedBackoff), MaxBackoff) + c.say(fmt.Sprintf("the search for setuid programs started %s did not complete after %s: %v", + now.Local().Format("15:04"), took.Round(time.Second), err)) + return + } + r := &setuidResult{Started: now, Finished: now.Add(took), Mounts: mounts, Found: found} + c.last, c.failed, c.backoff = r, nil, 0 + if err := c.keep(r); err != nil { + c.say("the search for setuid programs could not be kept, and judges from memory until the engine " + + "restarts: " + err.Error()) + } + c.say(fmt.Sprintf("the search for setuid programs started %s completed in %s: %d no package owns", + now.Local().Format("15:04"), took.Round(time.Second), len(found))) + }() +} + +func (c *SetuidCache) say(s string) { + if c.Say != nil { + c.Say(s) + } +} + +// covers says every filesystem a search would walk now was walked. +func covers(walked, now []string) bool { + for _, m := range now { + if !slices.Contains(walked, m) { + return false + } + } + return true +} + +// Wait is for a test: until the search running, if any, has finished. +func (c *SetuidCache) Wait() { c.wg.Wait() } + +// keep writes a complete search to Path: a new file beside it, readable by root alone, renamed over it. +func (c *SetuidCache) keep(r *setuidResult) error { + if c.Path == "" { + return nil + } + raw, err := json.Marshal(r) + if err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(c.Path), "."+filepath.Base(c.Path)+".*") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err := f.Write(raw); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), c.Path) +} + +// load reads the kept search, or nil: none kept, or one that cannot be believed, said why. +func (c *SetuidCache) load() *setuidResult { + if c.Path == "" { + return nil + } + r, err := keptSearch(c.Path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + c.say("the kept search for setuid programs is not read, and a new one runs: " + err.Error()) + } + return r +} + +// keptSearch reads a kept search, believing it only as a regular file, not followed through a link, owned by +// the engine's own account and writable by it alone, whose directory is the same: anything else could have been +// written by somebody other than root, and a search it says is no search. +func keptSearch(path string) (*setuidResult, error) { + if err := ownedAlone(filepath.Dir(path), true); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, err + } + if err := ownedAloneInfo(path, info, false); err != nil { + return nil, err + } + raw, err := io.ReadAll(io.LimitReader(f, 1<<20)) + if err != nil { + return nil, err + } + var r setuidResult + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("%s is not a kept search: %w", path, err) + } + if r.Started.IsZero() || r.Finished.Before(r.Started) || len(r.Mounts) == 0 { + return nil, fmt.Errorf("%s says no complete search", path) + } + return &r, nil +} + +func ownedAlone(path string, dir bool) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + return ownedAloneInfo(path, info, dir) +} + +func ownedAloneInfo(path string, info fs.FileInfo, dir bool) error { + if dir && !info.IsDir() || !dir && !info.Mode().IsRegular() { + return fmt.Errorf("%s is not a plain %s", path, map[bool]string{true: "directory", false: "file"}[dir]) + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("%s's owner could not be read", path) + } + if int(st.Uid) != os.Geteuid() { + return fmt.Errorf("%s is owned by uid %d, not the engine's %d", path, st.Uid, os.Geteuid()) + } + if info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("%s may be written by others than its owner (mode %o)", path, info.Mode().Perm()) + } + return nil +} diff --git a/internal/accounts/setuid_cache_test.go b/internal/accounts/setuid_cache_test.go new file mode 100644 index 0000000..c072b93 --- /dev/null +++ b/internal/accounts/setuid_cache_test.go @@ -0,0 +1,292 @@ +package accounts + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/rootsearch" +) + +// The search for setuid programs runs to completion and judges from its last complete result while it is fresh +// (novox/hq issue 361): never a bound that ends it, never a verdict from a search that did not finish or is too +// old, a new search after root acted, and the result kept beside the engine's state, believed only as root's. + +var at = time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC) + +var root = []string{"/"} + +// gate is a search that waits until released; each search started hands its context on started. found and +// err are read when it is released. +type gate struct { + release chan struct{} + started chan context.Context + found []string + err error +} + +func newGate() *gate { + return &gate{release: make(chan struct{}, 16), started: make(chan context.Context, 16)} +} + +func (g *gate) search(ctx context.Context, _ []string) ([]string, error) { + g.started <- ctx + <-g.release + return g.found, g.err +} + +func (g *gate) finish(c *SetuidCache) { g.release <- struct{}{}; c.Wait() } + +// starts is how many searches c started. +func starts(c *SetuidCache) int { + c.mu.Lock() + defer c.mu.Unlock() + return c.starts +} + +func notJudged(t *testing.T, err error, pending bool) *SetuidNotJudged { + t.Helper() + var nj *SetuidNotJudged + if !errors.As(err, &nj) || nj.Pending != pending { + t.Fatalf("want not judged (pending %v), got %v", pending, err) + } + want := ReasonIncomplete + if pending { + want = ReasonPending + } + if !strings.HasPrefix(err.Error(), want) { + t.Fatalf("the words start %q: %v", want, err) + } + return nj +} + +func TestTheSearchRunsToCompletionWithNoBound(t *testing.T) { + c := &SetuidCache{} + g := newGate() + if _, _, err := c.Look(at, root, g.search); err == nil { + t.Fatal("a look before the first search finished judged") + } + ctx := <-g.started + if _, ok := ctx.Deadline(); ok { + t.Fatal("the search was given a deadline: it must run until it has walked everything") + } + // Hours later, still running: still pending, never killed, never a second walk beside it. + _, _, err := c.Look(at.Add(5*time.Hour), root, g.search) + notJudged(t, err, true) + if starts(c) != 1 || ctx.Err() != nil { + t.Fatalf("a running search was started again or stopped: %d searches, %v", starts(c), ctx.Err()) + } + g.found = []string{"/opt/x"} + g.finish(c) + found, when, err := c.Look(at.Add(5*time.Hour), root, g.search) + if err != nil || len(found) != 1 || !when.Equal(at) { + t.Fatalf("the complete search judges, from its start: %v %v %v", found, when, err) + } +} + +func TestAFreshCompleteSearchJudgesUntilItIsDue(t *testing.T) { + c := &SetuidCache{} + g := newGate() + c.Look(at, root, g.search) + g.finish(c) + if _, _, err := c.Look(at.Add(rootsearch.Every-time.Minute), root, g.search); err != nil { + t.Fatalf("a complete search within a day: %v", err) + } + if starts(c) != 1 { + t.Fatalf("searched again before it was due: %d", starts(c)) + } + // Due: a new search starts, and the last complete one judges meanwhile. + if _, _, err := c.Look(at.Add(rootsearch.Every), root, g.search); err != nil { + t.Fatalf("while the daily search runs, the last complete one still judges: %v", err) + } + if starts(c) != 2 { + t.Fatalf("the daily search did not start: %d", starts(c)) + } + g.finish(c) +} + +func TestAStaleSearchIsNotJudgedNeverFree(t *testing.T) { + c := &SetuidCache{} + g := newGate() + c.Look(at, root, g.search) + g.finish(c) + c.Look(at.Add(rootsearch.Every), root, g.search) // the daily one, which does not finish + _, _, err := c.Look(at.Add(rootsearch.FreshFor), root, g.search) + nj := notJudged(t, err, true) + if !nj.Last.Equal(at) || !strings.Contains(err.Error(), "too old to judge") { + t.Fatalf("a stale search says when the last complete one started: %v", err) + } + g.finish(c) + if _, _, err := c.Look(at.Add(rootsearch.FreshFor), root, g.search); err != nil { + t.Fatalf("once the new search completed it judges again: %v", err) + } +} + +func TestAnIncompleteSearchIsNeverFree(t *testing.T) { + c := &SetuidCache{} + g := newGate() + g.err = errors.New("the search for setuid programs did not complete its walk") + c.Look(at, root, g.search) + g.finish(c) + _, _, err := c.Look(at.Add(time.Minute), root, g.search) + notJudged(t, err, false) + if starts(c) != 1 { + t.Fatalf("tried again within its back-off: %d", starts(c)) + } + c.Look(at.Add(FailedBackoff), root, g.search) + if starts(c) != 2 { + t.Fatalf("not tried again after its back-off: %d", starts(c)) + } + g.finish(c) + // A stale complete search and a failed new one: still not judged, never the stale one. + c = &SetuidCache{} + g = newGate() + c.Look(at, root, g.search) + g.finish(c) + g.err = errors.New("pacman could not say who owns the setuid programs found") + c.Look(at.Add(rootsearch.Every), root, g.search) + g.finish(c) + if _, _, err := c.Look(at.Add(rootsearch.Every+time.Hour), root, g.search); err != nil { + t.Fatalf("a failed search leaves the fresh complete one judging: %v", err) + } + _, _, err = c.Look(at.Add(rootsearch.FreshFor), root, g.search) + if err == nil { + t.Fatal("a stale search judged after the newer one failed") + } + g.finish(c) +} + +func TestAnApplyThatMovedRootSearchesAgain(t *testing.T) { + c := &SetuidCache{} + g := newGate() + c.Look(at, root, g.search) + g.finish(c) + c.SearchAgain() + if _, _, err := c.Look(at.Add(AgainAfter), root, g.search); err != nil { + t.Fatalf("the last complete search judges while the new one runs: %v", err) + } + if starts(c) != 2 { + t.Fatalf("an apply that moved root did not search again: %d", starts(c)) + } + // Asked again while one runs: one more after it, never two at once. + c.SearchAgain() + c.Look(at.Add(AgainAfter+time.Minute), root, g.search) + if starts(c) != 2 { + t.Fatalf("a second search started beside the running one: %d", starts(c)) + } + g.finish(c) + c.Look(at.Add(2*AgainAfter+time.Minute), root, g.search) + if starts(c) != 3 { + t.Fatalf("the ask made while a search ran was dropped: %d", starts(c)) + } + g.finish(c) + // A filesystem the last search did not walk asks for one too. + c.Look(at.Add(4*AgainAfter), []string{"/", "/srv"}, g.search) + if starts(c) != 4 { + t.Fatalf("a new filesystem was not searched: %d", starts(c)) + } + g.finish(c) + var nilCache *SetuidCache + nilCache.SearchAgain() +} + +func TestTheCompleteSearchIsKeptAndReadAfterARestart(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, SetuidSearchFile) + c := &SetuidCache{Path: path} + g := newGate() + g.found = []string{"/opt/x"} + c.Look(at, root, g.search) + g.finish(c) + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("kept readable by its owner alone: %v %v", info, err) + } + restarted := &SetuidCache{Path: path} + found, when, err := restarted.Look(at.Add(time.Hour), root, newGate().search) + if err != nil || len(found) != 1 || !when.Equal(at) { + t.Fatalf("a restarted engine judges from the kept search: %v %v %v", found, when, err) + } + restarted.Wait() + // And not once it is too old. + again := &SetuidCache{Path: path} + g2 := newGate() + if _, _, err := again.Look(at.Add(rootsearch.FreshFor), root, g2.search); err == nil { + t.Fatal("a kept search too old to judge judged") + } + g2.finish(again) +} + +func TestAKeptSearchNotRootsAloneIsNotRead(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, SetuidSearchFile) + c := &SetuidCache{Path: path} + g := newGate() + c.Look(at, root, g.search) + g.finish(c) + for name, spoil := range map[string]func() error{ + "writable by others": func() error { return os.Chmod(path, 0o666) }, + "a link": func() error { + other := filepath.Join(dir, "elsewhere.json") + if err := os.Rename(path, other); err != nil { + return err + } + return os.Symlink(other, path) + }, + "its directory writable by others": func() error { return os.Chmod(dir, 0o777) }, + } { + c = &SetuidCache{Path: path} + g = newGate() + c.Look(at, root, g.search) + g.finish(c) // a fresh, good one kept again + os.Chmod(dir, 0o700) + if err := spoil(); err != nil { + t.Fatal(err) + } + var said []string + r := &SetuidCache{Path: path, Say: func(s string) { said = append(said, s) }} + gr := newGate() + if _, _, err := r.Look(at.Add(time.Minute), root, gr.search); err == nil { + t.Fatalf("%s: a kept search that is not root's alone judged", name) + } + if len(said) == 0 { + t.Fatalf("%s: refusing the kept search said nothing", name) + } + gr.finish(r) + os.Chmod(dir, 0o700) + os.Remove(path) + } +} + +func TestAKeptSearchFromTheFutureIsNotFresh(t *testing.T) { + c := &SetuidCache{} + g := newGate() + c.Look(at, root, g.search) + g.finish(c) + if _, _, err := c.Look(at.Add(-time.Hour), root, g.search); err == nil { + t.Fatal("a clock set back an hour read a search as fresh") + } + g.finish(c) +} + +func TestTheIdleRunnerLowersItsCommand(t *testing.T) { + if _, err := exec.LookPath("nice"); err != nil { + t.Skip("no nice here") + } + out, err := Idle(t.Context(), "sh", "-c", "sleep 0.3; nice") + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(out) != "19" { + t.Fatalf("the search's commands run at nice %q, want 19", strings.TrimSpace(out)) + } + if _, err := Idle(t.Context(), "sh", "-c", "echo gone >&2; exit 1"); err == nil || + !strings.HasPrefix(err.Error(), "sh exited 1: gone") { + t.Fatalf("its failure is worded as CLocale's: %v", err) + } +} diff --git a/internal/accounts/ways.go b/internal/accounts/ways.go index 122b0cf..8d6c129 100644 --- a/internal/accounts/ways.go +++ b/internal/accounts/ways.go @@ -10,14 +10,12 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/novox/mesh-host/rootsearch" "io/fs" "os" "os/exec" "path/filepath" "sort" "strings" - "sync" "time" "golang.org/x/sys/unix" @@ -380,114 +378,16 @@ func contains(xs []string, s string) bool { return false } -// SetuidCache keeps the search for setuid programs, which walks every local filesystem, and runs it in the -// background (novox/hq ADR 0266, the third review): a disk too large to search in time must never stall a -// look, the health statement or the apply report. A look reads the last result the search left, and starts a -// new search when the last one is older than Every; a search that failed or did not finish is tried again -// after a back-off that doubles, from Every up to MaxBackoff. -type SetuidCache struct { - Every time.Duration - - mu sync.Mutex - wg sync.WaitGroup - at time.Time // when the last search that finished was started, on the caller's clock - found []string - running bool - startedAt time.Time - failedAt time.Time - failed error - backoff time.Duration -} - -// SearchBound is how long one search may run in the background — the whole search, the walk and the package -// manager's answers together (rootsearch.Bound, which the controller reads too); MaxBackoff the longest wait -// after failures. -const ( - SearchBound = rootsearch.Bound - MaxBackoff = 6 * time.Hour -) - -// SetuidNotJudged is the search's answer when it has no result to give: never run to its end yet, or failed -// every time so far. The judge says the way is not judged, never that there is none. -type SetuidNotJudged struct { - // Pending is true while the first search runs; false after one failed or did not finish. - Pending bool - Since time.Time - Err error -} - -func (e *SetuidNotJudged) Error() string { - if e.Pending { - return fmt.Sprintf("%s: the search for setuid programs, started at %s, has not finished yet", ReasonPending, - e.Since.Local().Format("15:04")) - } - return fmt.Sprintf("%s: the search for setuid programs did not finish (last tried at %s: %v); it is tried "+ - "again later", ReasonIncomplete, e.Since.Local().Format("15:04"), e.Err) -} - -// The reasons of a root verdict the setuid search could not answer. -const ( - ReasonPending = rootsearch.ReasonPending - ReasonIncomplete = rootsearch.ReasonIncomplete -) - -// Look answers the last result and when its search was started, starting a search in the background when one is -// due; it never waits for one. Nil c searches in place, for a caller that wants to. -// -// **One clock: the caller's.** A search's time is the `now` it was started at, never the wall clock read when it -// finished: due-ness compares a later `now` with it, and mixing the two clocks made a search never due again -// whenever the caller's clock ran behind the wall clock — the test with a fixed date failed on every branch from -// 2026-10-09 18:00 UTC (found while checking issue 356). Measured from the start, a result reads at most one -// search older than it is, which errs towards searching again. -func (c *SetuidCache) Look(now time.Time, search func(ctx context.Context) ([]string, error)) ([]string, time.Time, error) { - if c == nil { - ctx, cancel := context.WithTimeout(context.Background(), SearchBound) - defer cancel() - found, err := search(ctx) - return found, now, err - } - c.mu.Lock() - defer c.mu.Unlock() - due := !c.running && (c.at.IsZero() || now.Sub(c.at) >= c.Every) && - (c.failed == nil || now.Sub(c.failedAt) >= c.backoff) - if due { - c.running, c.startedAt = true, now - started := now - c.wg.Add(1) - go func() { - defer c.wg.Done() - ctx, cancel := context.WithTimeout(context.Background(), SearchBound) - found, err := search(ctx) - cancel() - c.mu.Lock() - defer c.mu.Unlock() - c.running = false - if err != nil { - c.failed, c.failedAt = err, started - c.backoff = min(max(2*c.backoff, c.Every), MaxBackoff) - return - } - c.found, c.at, c.failed, c.backoff = found, started, nil, 0 - }() - } - switch { - case !c.at.IsZero(): - return c.found, c.at, nil - case c.failed != nil: - return nil, time.Time{}, &SetuidNotJudged{Since: c.failedAt, Err: c.failed} - default: - return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: c.startedAt} - } -} - -// Wait is for a test: until the search running, if any, has finished. -func (c *SetuidCache) Wait() { c.wg.Wait() } - // setuidSearch is every setuid- or setgid-root regular file on the root filesystem that no installed package -// owns, found with find and asked of the package manager. Bounded by its caller (SearchBound, the whole search): -// a search that does not finish is an unanswered question, never "none". A walk find could not complete — it -// exited with an error — is incomplete too, unless every error was a file that vanished while it walked. +// owns, found with find and asked of the package manager, through the search's runner (Exec.SearchRun: idle +// priority on a serving engine). **No bound of its own** (novox/hq issue 361): it runs until it has walked +// everything, and only a context its caller ends stops it — a search that is stopped, or whose walk find could +// not complete (it exited with an error), is an unanswered question, never "none", unless every error was a +// file that vanished while it walked. func (e Exec) setuidSearch(ctx context.Context, mounts []string) ([]string, error) { + if e.SearchRun != nil { + e.Run = e.SearchRun + } // Each mount point a starting point of its own, -xdev keeping each to its own filesystem: every local // filesystem mounted without nosuid is searched once (the re-review of 2026-10-08), and a program setgid to // root's group counts whoever owns it. @@ -497,7 +397,7 @@ func (e Exec) setuidSearch(ctx context.Context, mounts []string) ([]string, erro "-2000", ")", ")", "-print") out, err := e.Run(ctx, "find", args...) if ctx.Err() != nil { - return nil, fmt.Errorf("the search for setuid programs did not finish within %s", SearchBound) + return nil, fmt.Errorf("the search for setuid programs was stopped before it finished: %w", ctx.Err()) } if err != nil && (strings.TrimSpace(out) == "" || !onlyVanished(err)) { return nil, fmt.Errorf("the search for setuid programs did not complete its walk: %w", err) @@ -715,18 +615,16 @@ func (e Exec) moreWays(ctx context.Context, account string, uid int, groups []st if len(mounts) == 0 { return nil, errors.New("no filesystem a setuid program could run from was found in /proc/mounts") } - // In the background, its last result served (SetuidCache): never a stall. No result yet is said beside - // the other ways, which are judged all the same. - unowned, searchedAt, searchErr := e.Cache.Look(now(), func(sctx context.Context) ([]string, error) { - return e.setuidSearch(sctx, mounts) - }) + // In the background, the last complete result served while it is fresh (SetuidCache): never a stall. No + // fresh complete result is said beside the other ways, which are judged all the same. + unowned, searchedAt, searchErr := e.Cache.Look(now(), mounts, e.setuidSearch) var notJudged *SetuidNotJudged if searchErr != nil && !errors.As(searchErr, ¬Judged) { return nil, searchErr } for _, p := range unowned { - ways = append(ways, "a setuid-root program no package owns: "+p+" (searched at "+ - searchedAt.Local().Format("15:04")+")") + ways = append(ways, "a setuid-root program no package owns: "+p+" (search started "+ + searchedAt.Local().Format("2006-01-02 15:04")+")") } // The kernel's protection of links: off, any account may link another's file where root then acts on it. diff --git a/internal/accounts/ways_test.go b/internal/accounts/ways_test.go index b62de60..69d8d19 100644 --- a/internal/accounts/ways_test.go +++ b/internal/accounts/ways_test.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/novox/mesh-host/rootsearch" "os/exec" "strings" "testing" @@ -132,56 +131,6 @@ func TestAFurtherQuestionUnansweredIsUnknown(t *testing.T) { } } -func TestTheSetuidSearchRunsInTheBackgroundAndIsKeptForItsInterval(t *testing.T) { - c := &SetuidCache{Every: time.Hour} - searched := 0 - release := make(chan struct{}) - search := func(context.Context) ([]string, error) { <-release; searched++; return []string{"/opt/x"}, nil } - at := time.Date(2026, 10, 8, 19, 0, 0, 0, time.UTC) - _, _, err := c.Look(at, search) - var nj *SetuidNotJudged - if !errors.As(err, &nj) || !nj.Pending || !strings.HasPrefix(err.Error(), ReasonPending) { - t.Fatalf("a look while the first search runs answers at once, not judged yet: %v", err) - } - if _, _, err := c.Look(at.Add(time.Minute), search); err == nil { - t.Fatal("still running: still not judged") - } - close(release) - c.Wait() - found, when, err := c.Look(at.Add(30*time.Minute), search) - if err != nil || len(found) != 1 || when.IsZero() { - t.Fatalf("the result once there: %v %v %v", found, when, err) - } - c.Look(at.Add(24*time.Hour), search) - c.Wait() - if searched != 2 { - t.Fatalf("searched %d times, want 2: once at first, once when the result was older than its interval", searched) - } -} - -func TestASearchThatFailsIsNotJudgedAndBacksOff(t *testing.T) { - c := &SetuidCache{Every: time.Hour} - tries := 0 - search := func(context.Context) ([]string, error) { tries++; return nil, errors.New("did not finish") } - at := time.Now() - c.Look(at, search) - c.Wait() - _, _, err := c.Look(at.Add(time.Minute), search) - var nj *SetuidNotJudged - if !errors.As(err, &nj) || nj.Pending || !strings.HasPrefix(err.Error(), ReasonIncomplete) { - t.Fatalf("a failed search is not judged (search incomplete): %v", err) - } - c.Wait() - if tries != 1 { - t.Fatalf("tried again within its back-off: %d", tries) - } - c.Look(at.Add(2*time.Hour), search) - c.Wait() - if tries != 2 { - t.Fatalf("tried again after its back-off: %d", tries) - } -} - func TestANotJudgedSearchStillSaysAWayFound(t *testing.T) { j := New(fakeReader{ways: []string{"in the group docker, which grants root"}, err: &SetuidNotJudged{Pending: true, Since: time.Now()}}) j.Set([]Account{{Module: "m", ID: "m.a", Name: "agent", Root: true}}) @@ -320,32 +269,6 @@ polkit.addRule(function(a, s) { /* isInGroup("wheel") */ return polkit.Result.YE } } -// The search's bound is the one the controller reads (rootsearch.Bound): a search may run its whole fifteen -// minutes, with no shorter limit inside it that would end every long walk incomplete (the review of 2026-10-09). -func TestTheSearchRunsToItsBoundWithNoShorterLimitInside(t *testing.T) { - if rootsearch.Bound != 15*time.Minute { - t.Fatalf("the shared bound is %s", rootsearch.Bound) - } - m := clean() - var left time.Duration - run := func(ctx context.Context, name string, args ...string) (string, error) { - if name == "find" { - if d, ok := ctx.Deadline(); ok { - left = time.Until(d) - } - } - return m.run(ctx, name, args...) - } - ctx, cancel := context.WithTimeout(t.Context(), SearchBound) - defer cancel() - if _, err := (Exec{Run: run}).setuidSearch(ctx, []string{"/"}); err != nil { - t.Fatal(err) - } - if left < SearchBound-time.Minute { - t.Fatalf("find was given %s, less than the search's bound of %s", left, SearchBound) - } -} - // The package manager is asked about every path found in one call, and answers each: owned, or owned by nothing. func TestThePackageManagerIsAskedOnceForEveryPathFound(t *testing.T) { m := clean() diff --git a/internal/apply/root_moved.go b/internal/apply/root_moved.go new file mode 100644 index 0000000..af9bca2 --- /dev/null +++ b/internal/apply/root_moved.go @@ -0,0 +1,46 @@ +package apply + +import ( + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// rootPaths are the files and directories through which an account comes to root, or by which it is told who it +// is: the user and group databases, sudo's, doas's and polkit's rules. A file written there changes what the +// judge of an agent account reads (novox/hq ADR 0266). +var rootPaths = []string{"/etc/passwd", "/etc/shadow", "/etc/group", "/etc/gshadow", "/etc/sudoers", + "/etc/sudoers.d", "/etc/doas.conf", "/etc/doas.d", "/etc/polkit-1", "/usr/share/polkit-1"} + +// MovesRoot says an apply changed something through which root's reach could have changed — an account, a +// package, an action run as root, or a file in rootPaths (novox/hq issue 361). The engine then asks for a new +// search for setuid programs: root acted, and a complete search is sound only until root acts. Outcomes that +// moved nothing (unchanged, held, waiting, held still) do not count. +func MovesRoot(outcomes []Outcome) bool { + for _, o := range outcomes { + switch o.Action { + case "unchanged", "held", "waiting", heldStill: + continue + } + switch declaration.Type(o.Type) { + case declaration.TypeUser, declaration.TypePackage, declaration.TypeAction: + return true + case declaration.TypeFile, declaration.TypeDirectory, declaration.TypeArchive: + if underRootPath(o.Target) { + return true + } + } + } + return false +} + +func underRootPath(target string) bool { + t := filepath.Clean(target) + for _, p := range rootPaths { + if t == p || strings.HasPrefix(t, p+"/") { + return true + } + } + return false +} diff --git a/internal/apply/root_moved_test.go b/internal/apply/root_moved_test.go new file mode 100644 index 0000000..683f088 --- /dev/null +++ b/internal/apply/root_moved_test.go @@ -0,0 +1,29 @@ +package apply + +import "testing" + +// An apply that moved what root controls asks for a new search for setuid programs (novox/hq issue 361); one +// that moved nothing of it does not. +func TestAnApplyMovesRootThroughAnAccountAPackageAnActionOrSudosRules(t *testing.T) { + for _, c := range []struct { + o Outcome + want bool + }{ + {Outcome{Type: "user", Target: "nox", Action: "created"}, true}, + {Outcome{Type: "user", Target: "nox", Action: "unchanged"}, false}, + {Outcome{Type: "package", Target: "telegram", Action: "created"}, true}, + {Outcome{Type: "package", Target: "telegram", Action: "removed"}, true}, + {Outcome{Type: "action", Target: "setup", Action: "updated"}, true}, + {Outcome{Type: "file", Target: "/etc/sudoers.d/mesh", Action: "updated"}, true}, + {Outcome{Type: "file", Target: "/etc/group", Action: "corrected"}, true}, + {Outcome{Type: "directory", Target: "/etc/polkit-1/rules.d", Action: "created"}, true}, + {Outcome{Type: "file", Target: "/etc/sudoers.dx", Action: "updated"}, false}, + {Outcome{Type: "file", Target: "/etc/nginx/nginx.conf", Action: "updated"}, false}, + {Outcome{Type: "container", Target: "plex", Action: "updated"}, false}, + {Outcome{Type: "package", Target: "x", Action: heldStill}, false}, + } { + if got := MovesRoot([]Outcome{c.o}); got != c.want { + t.Errorf("%+v: moves root %v, want %v", c.o, got, c.want) + } + } +} diff --git a/rootsearch/rootsearch.go b/rootsearch/rootsearch.go index a1c70c0..7692529 100644 --- a/rootsearch/rootsearch.go +++ b/rootsearch/rootsearch.go @@ -1,19 +1,43 @@ // Package rootsearch holds what the node-engine's search for setuid programs and the controller that reads its -// verdicts must agree on (novox/hq ADR 0266): how long one search may run, and the words a verdict starts with -// while it has no answer. Public, so the controller imports these rather than keeps copies that could drift. +// verdicts must agree on (novox/hq ADR 0266, issue 361): how often a full search runs, how long a complete one +// judges, how long the controller stays quiet while one runs, and the words a verdict starts with while it has +// no answer. Public, so the controller imports these rather than keeps copies that could drift. +// +// **Why a complete search may judge for a day and more.** The search looks for a setuid- or setgid-root program +// no package owns, a way for the agent account to become root. Only root can make such a file: the agent account +// can neither set the setuid bit on a file root owns nor give a file it owns to root. So a search that walked +// everything stays sound until root acts — and root's acts through the mesh (an apply that changes an account, a +// group, a sudo rule, a package or runs an action) start a new search at once. What is left is root acting by +// hand, which is the operator; FreshFor bounds how long that goes unseen. +// +// **And why no bound on one search.** On a machine with millions of files the walk takes longer than any bound +// worth stating (the control-node's did not finish in fifteen minutes, issue 361), and a search ended early +// judges nothing, so a bound made such a machine never judged. The search runs to its end at idle priority, +// once at a time; an incomplete search is never "free". package rootsearch import "time" -// Bound is how long one search for setuid programs may run. It governs the whole search, the walk and the -// package manager's answers together; the controller does not raise an agent account as not judged while the -// first search after a start is within it. -const Bound = 15 * time.Minute +const ( + // Every is how often a full search runs at most, counted from the start of the last complete one; an + // apply that changes what root controls starts one sooner. + Every = 24 * time.Hour + // FreshFor is how long a complete search judges, counted from its start: after it, the verdict is "not + // judged yet" until a newer search completes. Every plus Quiet, so a daily search that finishes within + // the controller's quiet leaves no hour unjudged. + FreshFor = Every + Quiet + // Quiet is how long the controller raises nothing while an agent account waits for a search, counted + // from when it first saw it waiting: the longest a full search is expected to take at idle priority on + // the largest machine. Past it, waiting is itself the urgent condition. It never makes a machine free. + Quiet = 12 * time.Hour +) // The reasons of a root verdict the search could not answer yet. const ( - // ReasonPending starts the reason while the first search since the engine started runs. + // ReasonPending starts the reason while no complete search judges: none has finished since the engine's + // state was kept, or the last one is older than FreshFor, and one runs. ReasonPending = "not judged yet (search running)" - // ReasonIncomplete starts the reason when no search has finished: one failed or ran out its bound. + // ReasonIncomplete starts the reason when no complete search judges and the last one failed: it is tried + // again after a back-off. ReasonIncomplete = "not judged (search incomplete)" )