package accounts import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "slices" "strings" "sync" "syscall" "time" "github.com/novox/mesh-host/rootsearch" ) // SetuidCache keeps the search for setuid programs, which walks every local filesystem, and runs it in the // background (novox/hq ADR 0266, issue 361): a look never waits for it, and the search itself is never cut // short. // // - **A verdict comes only from a complete search, while it is fresh**: one that walked everything and had the // package manager answer for every path found, started less than FreshFor ago. Before the first completes, // or once the last is older, the answer is "not judged yet" (ReasonPending) while a search runs, and "not // judged" (ReasonIncomplete) after one failed. Never "none found" from a search that did not finish. // - **It runs to completion**, once at a time, with no bound, on a thread of its own at idle priority (Lower): // a fresh one when the last complete one is Every old, when SearchAgain asks (an apply changed what root // controls), or when a filesystem appeared that the last one did not walk. One that fails is tried again // after a back-off that doubles from FailedBackoff up to MaxBackoff. // - **Resumable**: the walk's progress is kept in ProgressPath every CheckpointEvery, and a search started // again — after an engine restart, an update, a reboot or a failure — carries on from it while it is // younger than FreshFor. A search SearchAgain asked for never resumes an older walk: it starts from nothing. // - **Kept** in Path, beside the engine's other state, with when it started and finished: an engine restart // serves the last complete search rather than judging nothing until a new walk ends. // // Why a complete search stays sound for a day and more, and a resumed walk is sound, is said in the rootsearch // package and setuid_walk.go: only root makes a setuid-root file, never the agent account. type SetuidCache struct { // Every and FreshFor are rootsearch's when zero. Every, FreshFor time.Duration // Path is the file the last complete search is kept in, and ProgressPath the walk under way; empty keeps // either in memory only. The engine's own state directory, which only root writes: a file there that is // not a regular file the engine's account owns and alone may write is not read (keptFile). Path, ProgressPath string // Lower is called on the search's own goroutine before it walks (LowerThisThread on a serving engine); a // priority it cannot lower fails the search rather than walk the disks at full priority. Nil lowers nothing. Lower func() error // Say is where a search's end and a kept file refused are said; nil says nothing. Say func(string) mu sync.Mutex wg sync.WaitGroup loaded bool last *setuidResult // the last complete search unfinished *SetuidProgress // a walk that did not finish, to resume progress *SearchProgress // how far the running search got, as of its last checkpoint running bool gen int // counts SearchNow: a search started under an older one is thrown away cancel context.CancelFunc // ends the running search's walk startedAt time.Time // when the search running, or the last one, was started (a resumed one: its start) attemptAt time.Time // when the running or last search was started or resumed endedAt time.Time // when the last search ended, on the same clock again bool // asked to search again since the running or last one started failedAt time.Time failed error backoff time.Duration starts int // how many searches were started or resumed, for a test } // setuidResult is one complete search, as kept. type setuidResult struct { // Started and Finished are on the caller's clock: Finished is Started plus how long the search took. Started time.Time `json:"started"` Finished time.Time `json:"finished"` // Mounts are the filesystems it walked; Found the setuid-root programs no package owns. Mounts Paths `json:"mounts"` Found Paths `json:"found"` } const ( // FailedBackoff is the first wait after a search failed, doubling to MaxBackoff. FailedBackoff = time.Hour MaxBackoff = 6 * time.Hour // AgainAfter is the least time between the starts of two searches that SearchAgain or a new filesystem // asked for: a run of applies does not walk the disks back to back. AgainAfter = time.Hour // clockSkew is how far in the future a kept search may say it started and still judge: a clock set back // further makes it no verdict, never a fresh one. clockSkew = 5 * time.Minute // SetuidSearchFile and SetuidProgressFile are the files' names beside the engine's state: the last // complete search, and the walk under way. SetuidSearchFile = "setuid-search.json" SetuidProgressFile = "setuid-search-progress.json" ) // SetuidNotJudged is the search's answer when no complete search judges: none has finished yet, the last one is // older than FreshFor, or every one since failed. The judge says the way is not judged, never that there is none. type SetuidNotJudged struct { // Pending is true while a search runs; false after one failed. Pending bool Since time.Time Err error // Last is when the last complete search started, when there is one too old to judge. Last time.Time // Progress is how far the running search got, as of its last checkpoint: nil when none runs. Progress *SearchProgress } // SearchProgress is how far a running search got, as of its last checkpoint, for anybody to read (novox/hq // issue 361): counts, times and mount points only. **Never a path the walk visited or found**: the cursor and // the files found are under accounts' homes as often as not, and stay in the engine's state and journal. type SearchProgress struct { // Started is when the search began, on the cache's clock. Started time.Time // Filesystems is how many it walks; Done how many it has walked to their end; Walking the mount point of // the one it walks now, empty when every one is walked. Filesystems, Done int Walking string // Visited is how many entries it has gone past; Found how many setuid- or setgid-root files it found so // far, before the package manager is asked about them. Visited int64 Found int // AtLeast says Visited is a lower bound: the walk was begun by an engine that did not count. AtLeast bool // Kept is when its progress was last kept, zero before the first time. Kept time.Time // Finished is every filesystem walked to its end, in order, and how long its walk took. Finished []SearchedFilesystem } // SearchedFilesystem is one filesystem a search walked to its end. Took is negative when not known: walked by // an engine before it counted. type SearchedFilesystem struct { Mount string Took time.Duration } // progressOf is p as anybody may read it. Counts and mount points, no I/O. func progressOf(p *SetuidProgress, kept time.Time) *SearchProgress { sp := &SearchProgress{Started: p.Started, Filesystems: len(p.Mounts), Done: len(p.Done), Visited: p.Visited, Found: len(p.Found), Kept: kept, AtLeast: !p.Counted} for i, m := range p.Done { took := time.Duration(-1) if i < len(p.Took) { took = p.Took[i] } sp.Finished = append(sp.Finished, SearchedFilesystem{Mount: m, Took: took}) } for _, m := range p.Mounts { if !slices.Contains(p.Done, m) { sp.Walking = m break } } return sp } // Words are the progress in one line, as the verdict says it beside "not judged yet". func (sp *SearchProgress) Words() string { if sp == nil { return "" } s := fmt.Sprintf("progress: %d of %d filesystems walked", sp.Done, sp.Filesystems) if sp.Walking != "" { s += ", walking " + sp.Walking } atLeast := "" if sp.AtLeast { atLeast = "at least " } s += fmt.Sprintf(", %s%d entries visited, %d setuid- or setgid-root files found so far", atLeast, sp.Visited, sp.Found) if sp.Kept.IsZero() { s += ", not kept yet" } else { s += ", kept " + sp.Kept.Local().Format("2006-01-02 15:04:05") } var took []string for _, f := range sp.Finished { if f.Took < 0 { took = append(took, f.Mount+" in a time not known") } else { took = append(took, f.Mount+" in "+f.Took.Round(time.Second).String()) } } if len(took) > 0 { s += "; walked " + strings.Join(took, ", ") } return s } func (e *SetuidNotJudged) Error() string { stale := "" if !e.Last.IsZero() { stale = fmt.Sprintf("; the last complete one, started %s, is too old to judge", e.Last.Local().Format("2006-01-02 15:04")) } if e.Pending { return fmt.Sprintf("%s: the search for setuid programs, started %s, has not finished yet%s", ReasonPending, e.Since.Local().Format("2006-01-02 15:04"), stale) } return fmt.Sprintf("%s: the search for setuid programs did not complete (last tried %s: %v); it is tried "+ "again later%s", ReasonIncomplete, e.Since.Local().Format("2006-01-02 15:04"), e.Err, stale) } // The reasons of a root verdict the setuid search could not answer. const ( ReasonPending = rootsearch.ReasonPending ReasonIncomplete = rootsearch.ReasonIncomplete ) func (c *SetuidCache) every() time.Duration { if c.Every > 0 { return c.Every } return rootsearch.Every } func (c *SetuidCache) freshFor() time.Duration { if c.FreshFor > 0 { return c.FreshFor } return rootsearch.FreshFor } // SearchAgain asks for a new search at the next look: an apply changed what root controls (an account, a group, // a sudo rule, a package, an action). The last complete search keeps judging until it is too old: only root acts // through an apply, and the agent account cannot make a setuid-root file. Nil c is nothing to ask. func (c *SetuidCache) SearchAgain() { if c == nil { return } c.mu.Lock() defer c.mu.Unlock() c.again = true // The walk under way, or kept, started before root acted: never carried on, even by an engine that // restarts before the next search starts (the review of 2026-10-10). It keeps no progress from now on. c.unfinished = nil c.forgetProgress() } // SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has // the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the // operator changed by hand what the last one found, which no apply says. Until the new one completes the // verdict is "not judged yet". Asked at the controller's terminal alone (`nox node setuid-search `): a // search can never make a machine free wrongly, but asked again and again it would keep a machine unjudged // and its disks busy. by is who asked, for the journal. Nil c has nothing to throw away. func (c *SetuidCache) SearchNow(by string) string { if c == nil { return "this engine keeps no search for setuid programs" } c.mu.Lock() defer c.mu.Unlock() c.loaded = true // nothing kept is read after this: it is thrown away what := "no complete search was kept" if c.last != nil { what = "the complete search started " + c.last.Started.Local().Format("2006-01-02 15:04") + " is thrown away" } if c.running { what += ", and the search running since " + c.startedAt.Local().Format("2006-01-02 15:04") + " is ended" if c.cancel != nil { c.cancel() } } c.gen++ c.last, c.unfinished, c.progress = nil, nil, nil c.failed, c.failedAt, c.backoff, c.again = nil, time.Time{}, 0, false c.forgetProgress() if c.Path != "" { if err := os.Remove(c.Path); err != nil && !errors.Is(err, fs.ErrNotExist) { c.say("the search for setuid programs could not forget the complete search it kept: " + err.Error()) } } said := "a new search for setuid programs starts at the next look, asked by " + by + ": " + what + "; until it completes, whether an account can become root through one is not judged" c.say(said) return said } // SetuidSearch is one search: it carries p on to its end — walking what p has not walked, calling checkpoint // as it goes so its progress can be kept — and answers the setuid-root programs no package owns. type SetuidSearch func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error) // Look answers the setuid-root programs no package owns as the last complete search found them, and when it // started — while it is fresh — and starts a search in the background when one is due; it never waits for one. // mounts are the filesystems a new search walks. Nil c searches in place, for a caller that wants to. // // **One clock: the caller's.** A search's time is the `now` it was started at, never the wall clock read when it // finished: due-ness compares a later `now` with it, and mixing the two clocks made a search never due again // whenever the caller's clock ran behind the wall clock (found while checking issue 356). Measured from the // start, a result reads at most one search older than it is, which errs towards not judging. func (c *SetuidCache) Look(now time.Time, mounts []string, search SetuidSearch) ([]string, time.Time, error) { if c == nil { p := &SetuidProgress{Started: now, Mounts: slices.Clone(mounts), Counted: true} found, err := search(context.Background(), p, func() {}) return found, now, err } c.mu.Lock() defer c.mu.Unlock() if !c.loaded { c.loaded = true c.last = c.loadResult() c.unfinished = c.loadProgress() } var age time.Duration if c.last != nil { age = now.Sub(c.last.Started) } // Fresh only while it walked every filesystem a search would walk now: one mounted since is not judged // by a search that never saw it. fresh := c.last != nil && age > -clockSkew && age < c.freshFor() && covers(c.last.Mounts, mounts) // Asked for (an apply, a new filesystem): at most once an AgainAfter, counted from when the last search // ended, so a walk of hours is not followed by another at once, apply after apply. gap := now.Sub(c.attemptAt) if !c.endedAt.IsZero() { gap = now.Sub(c.endedAt) } due := !c.running && (c.failed == nil || now.Sub(c.failedAt) >= c.backoff) && (!fresh || age >= c.every() || (c.again && (c.attemptAt.IsZero() || gap >= AgainAfter))) if due { c.start(now, mounts, search) } switch { case fresh: return slices.Clone(c.last.Found), c.last.Started, nil case c.running: return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: c.startedAt, Last: c.lastStarted(), Progress: c.progress} case c.failed != nil: return nil, time.Time{}, &SetuidNotJudged{Since: c.failedAt, Err: c.failed, Last: c.lastStarted()} default: // Not fresh, not running, never failed: cannot be, as not fresh is due. Never a verdict. return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: now, Last: c.lastStarted()} } } func (c *SetuidCache) lastStarted() time.Time { if c.last == nil { return time.Time{} } return c.last.Started } // resumable is the walk that did not finish, when a search may carry it on: younger than FreshFor, and no // SearchAgain since — an apply that moved root's things wants a walk that starts after it. Anything else is // thrown away. c.mu held. func (c *SetuidCache) resumable(now time.Time) *SetuidProgress { p := c.unfinished c.unfinished = nil if p == nil { return nil } age := now.Sub(p.Started) if c.again || age <= -clockSkew || age >= c.freshFor() { c.forgetProgress() return nil } return p } // start runs one search in the background, with no bound: a walk that did not finish carried on, or a new one. // c.mu held. func (c *SetuidCache) start(now time.Time, mounts []string, search SetuidSearch) { p := c.resumable(now) resumed := p != nil if p == nil { p = &SetuidProgress{ID: fmt.Sprintf("%x", now.UnixNano()), Started: now, Mounts: slices.Clone(mounts), Cursor: map[string]string{}, Counted: true} } c.running, c.startedAt, c.attemptAt, c.again = true, p.Started, now, false c.starts++ // What a look says until the first checkpoint: where the kept walk was, or nothing walked. kept := p.Kept c.progress = progressOf(p, kept) if resumed { c.say(fmt.Sprintf("the search for setuid programs started %s carries on where it was (%d of %d "+ "filesystems walked)", p.Started.Local().Format("2006-01-02 15:04"), len(p.Done), len(p.Mounts))) } began := time.Now() gen := c.gen ctx, cancel := context.WithCancel(context.Background()) c.cancel = cancel c.wg.Add(1) go func() { defer c.wg.Done() var found []string var err error if c.Lower != nil { // The goroutine stays on its lowered thread until it ends, and the thread ends with it. if lerr := c.Lower(); lerr != nil { err = fmt.Errorf("its priority could not be lowered, so it did not walk: %w", lerr) } } if err == nil { unkept := false found, err = search(ctx, p, func() { c.mu.Lock() superseded := c.again || c.gen != gen c.mu.Unlock() if !superseded { // root acted since this walk started: its progress is not to be carried on at := time.Now() p.Kept = at kerr := c.keepProgress(p) // Superseded while it wrote (SearchAgain, SearchNow), after they forgot the kept walk: forgotten // again here, so no engine restarted before this walk ends carries it on (the review of // 2026-10-10). Written outside the lock, so a look never waits on the disk. c.mu.Lock() if c.again || c.gen != gen { c.forgetProgress() } c.mu.Unlock() if kerr != nil { p.Kept = kept // not kept: the file still says the time before if !unkept { unkept = true c.say("the search for setuid programs could not keep how far it got, and starts over " + "if the engine stops: " + kerr.Error()) } } else { kept = at } } // What a look says from now: computed from p here, on the walk's own goroutine, never read // by a look while the walk changes it. sp := progressOf(p, kept) c.mu.Lock() if c.gen == gen { c.progress = sp } c.mu.Unlock() }) } took := time.Since(began) cancel() c.mu.Lock() defer c.mu.Unlock() c.running, c.progress, c.cancel = false, nil, nil c.endedAt = now.Add(took) if c.gen != gen { // Thrown away by SearchNow while it ran: neither a result nor a failure, and nothing to carry on. The // next look starts the search asked for. c.endedAt = time.Time{} // A checkpoint that read its generation just before SearchNow may have kept this walk again. c.forgetProgress() c.say(fmt.Sprintf("the search for setuid programs started %s was ended, as asked (%s this time)", p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second))) return } if err != nil { // The back-off counts from when it failed. c.failed, c.failedAt = err, c.endedAt c.backoff = min(max(2*c.backoff, FailedBackoff), MaxBackoff) if !c.again { // superseded by an ask to search again: never carried on c.unfinished = p } c.say(fmt.Sprintf("the search for setuid programs started %s did not complete (%s this time): %v", p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), err)) return } r := &setuidResult{Started: p.Started, Finished: now.Add(took), Mounts: p.Mounts, Found: found} c.last, c.failed, c.backoff = r, nil, 0 if err := c.keepResult(r); err != nil { c.say("the search for setuid programs could not be kept, and judges from memory until the engine " + "restarts: " + err.Error()) } c.forgetProgress() // The paths in the engine's journal alone: the verdict says one in a home by count. named := "" if len(found) > 0 { named = ": " + strings.Join(found, ", ") } c.say(fmt.Sprintf("the search for setuid programs started %s completed (%s this time): %d no package "+ "owns%s", p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), len(found), named)) }() } func (c *SetuidCache) say(s string) { if c.Say != nil { c.Say(s) } } // covers says every filesystem a search would walk now was walked. func covers(walked, now []string) bool { for _, m := range now { if !slices.Contains(walked, m) { return false } } return true } // Wait is for a test: until the search running, if any, has finished. func (c *SetuidCache) Wait() { c.wg.Wait() } func (c *SetuidCache) keepResult(r *setuidResult) error { return keepFile(c.Path, r) } func (c *SetuidCache) keepProgress(p *SetuidProgress) error { return keepFile(c.ProgressPath, p) } // forgetProgress removes the kept walk: finished, or thrown away. func (c *SetuidCache) forgetProgress() { if c.ProgressPath == "" { return } if err := os.Remove(c.ProgressPath); err != nil && !errors.Is(err, fs.ErrNotExist) { c.say("the search for setuid programs could not forget the walk it kept: " + err.Error()) } } // keepFile writes v to path: a new file beside it, readable by its owner alone, renamed over it — a link at // path is replaced, never followed. Empty path keeps nothing. func keepFile(path string, v any) error { if path == "" { return nil } raw, err := json.Marshal(v) if err != nil { return err } f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") if err != nil { return err } defer os.Remove(f.Name()) if _, err := f.Write(raw); err != nil { f.Close() return err } if err := f.Sync(); err != nil { f.Close() return err } if err := f.Close(); err != nil { return err } return os.Rename(f.Name(), path) } // loadResult reads the kept complete search, or nil: none kept, or one that cannot be believed, said why. func (c *SetuidCache) loadResult() *setuidResult { var r setuidResult if !c.loadFile(c.Path, "complete search", &r) { return nil } if r.Started.IsZero() || r.Finished.Before(r.Started) || len(r.Mounts) == 0 { c.say("the kept search for setuid programs says no complete search, and a new one runs") return nil } return &r } // loadProgress reads the kept walk, or nil. func (c *SetuidCache) loadProgress() *SetuidProgress { var p SetuidProgress if !c.loadFile(c.ProgressPath, "walk under way", &p) { return nil } if p.Started.IsZero() || len(p.Mounts) == 0 { c.say("the kept walk for setuid programs says no walk, and it starts over") c.forgetProgress() return nil } if p.Cursor == nil { p.Cursor = map[string]string{} } return &p } func (c *SetuidCache) loadFile(path, what string, v any) bool { if path == "" { return false } raw, err := keptFile(path) if errors.Is(err, fs.ErrNotExist) { return false } if err == nil { if err = json.Unmarshal(raw, v); err != nil { err = fmt.Errorf("%s is not JSON: %w", path, err) } } if err != nil { c.say(fmt.Sprintf("the kept %s for setuid programs is not read, and a new one runs: %v", what, err)) return false } return true } // keptFile reads a kept file, believing it only as a regular file, not followed through a link, owned by the // engine's own account and writable by it alone, in a directory that is the same: anything else could have been // written by somebody other than root, and a search it says is no search. func keptFile(path string) ([]byte, error) { if err := ownedAlone(filepath.Dir(path), true); err != nil { return nil, err } f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) if err != nil { return nil, err } defer f.Close() info, err := f.Stat() if err != nil { return nil, err } if err := ownedAloneInfo(path, info, false); err != nil { return nil, err } return io.ReadAll(io.LimitReader(f, 64<<20)) } func ownedAlone(path string, dir bool) error { info, err := os.Lstat(path) if err != nil { return err } return ownedAloneInfo(path, info, dir) } func ownedAloneInfo(path string, info fs.FileInfo, dir bool) error { if dir && !info.IsDir() || !dir && !info.Mode().IsRegular() { return fmt.Errorf("%s is not a plain %s", path, map[bool]string{true: "directory", false: "file"}[dir]) } st, ok := info.Sys().(*syscall.Stat_t) if !ok { return fmt.Errorf("%s's owner could not be read", path) } if int(st.Uid) != os.Geteuid() { return fmt.Errorf("%s is owned by uid %d, not the engine's %d", path, st.Uid, os.Geteuid()) } if info.Mode().Perm()&0o022 != 0 { return fmt.Errorf("%s may be written by others than its owner (mode %o)", path, info.Mode().Perm()) } return nil }