mesh-host/internal/accounts/setuid_cache.go

package accounts

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/novox/mesh-host/rootsearch"
)

// SetuidCache keeps the search for setuid programs, which walks every local filesystem, and runs it in the
// background (novox/hq ADR 0266, issue 361): a look never waits for it, and the search itself is never cut
// short.
//
//   - **A verdict comes only from a complete search, while it is fresh**: one that walked everything and had the
//     package manager answer for every path found, started less than FreshFor ago. Before the first completes,
//     or once the last is older, the answer is "not judged yet" (ReasonPending) while a search runs, and "not
//     judged" (ReasonIncomplete) after one failed. Never "none found" from a search that did not finish.
//   - **It runs to completion**, once at a time, with no bound, on a thread of its own at idle priority (Lower):
//     a fresh one when the last complete one is Every old, when SearchAgain asks (an apply changed what root
//     controls), or when a filesystem appeared that the last one did not walk. One that fails is tried again
//     after a back-off that doubles from FailedBackoff up to MaxBackoff.
//   - **Resumable**: the walk's progress is kept in ProgressPath every CheckpointEvery, and a search started
//     again — after an engine restart, an update, a reboot or a failure — carries on from it while it is
//     younger than FreshFor. A search SearchAgain asked for never resumes an older walk: it starts from nothing.
//   - **Kept** in Path, beside the engine's other state, with when it started and finished: an engine restart
//     serves the last complete search rather than judging nothing until a new walk ends.
//
// Why a complete search stays sound for a day and more, and a resumed walk is sound, is said in the rootsearch
// package and setuid_walk.go: only root makes a setuid-root file, never the agent account.
type SetuidCache struct {
	// Every and FreshFor are rootsearch's when zero.
	Every, FreshFor time.Duration
	// Path is the file the last complete search is kept in, and ProgressPath the walk under way; empty keeps
	// either in memory only. The engine's own state directory, which only root writes: a file there that is
	// not a regular file the engine's account owns and alone may write is not read (keptFile).
	Path, ProgressPath string
	// Lower is called on the search's own goroutine before it walks (LowerThisThread on a serving engine); a
	// priority it cannot lower fails the search rather than walk the disks at full priority. Nil lowers nothing.
	Lower func() error
	// Say is where a search's end and a kept file refused are said; nil says nothing.
	Say func(string)

	mu         sync.Mutex
	wg         sync.WaitGroup
	loaded     bool
	last       *setuidResult   // the last complete search
	unfinished *SetuidProgress // a walk that did not finish, to resume
	progress   *SearchProgress // how far the running search got, as of its last checkpoint
	running    bool
	gen        int                // counts SearchNow: a search started under an older one is thrown away
	cancel     context.CancelFunc // ends the running search's walk
	startedAt  time.Time          // when the search running, or the last one, was started (a resumed one: its start)
	attemptAt  time.Time          // when the running or last search was started or resumed
	endedAt    time.Time          // when the last search ended, on the same clock
	again      bool               // asked to search again since the running or last one started
	failedAt   time.Time
	failed     error
	backoff    time.Duration
	starts     int // how many searches were started or resumed, for a test
}

// setuidResult is one complete search, as kept.
type setuidResult struct {
	// Started and Finished are on the caller's clock: Finished is Started plus how long the search took.
	Started  time.Time `json:"started"`
	Finished time.Time `json:"finished"`
	// Mounts are the filesystems it walked; Found the setuid-root programs no package owns.
	Mounts Paths `json:"mounts"`
	Found  Paths `json:"found"`
}

const (
	// FailedBackoff is the first wait after a search failed, doubling to MaxBackoff.
	FailedBackoff = time.Hour
	MaxBackoff    = 6 * time.Hour
	// AgainAfter is the least time between the starts of two searches that SearchAgain or a new filesystem
	// asked for: a run of applies does not walk the disks back to back.
	AgainAfter = time.Hour
	// clockSkew is how far in the future a kept search may say it started and still judge: a clock set back
	// further makes it no verdict, never a fresh one.
	clockSkew = 5 * time.Minute
	// SetuidSearchFile and SetuidProgressFile are the files' names beside the engine's state: the last
	// complete search, and the walk under way.
	SetuidSearchFile   = "setuid-search.json"
	SetuidProgressFile = "setuid-search-progress.json"
)

// SetuidNotJudged is the search's answer when no complete search judges: none has finished yet, the last one is
// older than FreshFor, or every one since failed. The judge says the way is not judged, never that there is none.
type SetuidNotJudged struct {
	// Pending is true while a search runs; false after one failed.
	Pending bool
	Since   time.Time
	Err     error
	// Last is when the last complete search started, when there is one too old to judge.
	Last time.Time
	// Progress is how far the running search got, as of its last checkpoint: nil when none runs.
	Progress *SearchProgress
}

// SearchProgress is how far a running search got, as of its last checkpoint, for anybody to read (novox/hq
// issue 361): counts, times and mount points only. **Never a path the walk visited or found**: the cursor and
// the files found are under accounts' homes as often as not, and stay in the engine's state and journal.
type SearchProgress struct {
	// Started is when the search began, on the cache's clock.
	Started time.Time
	// Filesystems is how many it walks; Done how many it has walked to their end; Walking the mount point of
	// the one it walks now, empty when every one is walked.
	Filesystems, Done int
	Walking           string
	// Visited is how many entries it has gone past; Found how many setuid- or setgid-root files it found so
	// far, before the package manager is asked about them.
	Visited int64
	Found   int
	// AtLeast says Visited is a lower bound: the walk was begun by an engine that did not count.
	AtLeast bool
	// Kept is when its progress was last kept, zero before the first time.
	Kept time.Time
	// Finished is every filesystem walked to its end, in order, and how long its walk took.
	Finished []SearchedFilesystem
}

// SearchedFilesystem is one filesystem a search walked to its end. Took is negative when not known: walked by
// an engine before it counted.
type SearchedFilesystem struct {
	Mount string
	Took  time.Duration
}

// progressOf is p as anybody may read it. Counts and mount points, no I/O.
func progressOf(p *SetuidProgress, kept time.Time) *SearchProgress {
	sp := &SearchProgress{Started: p.Started, Filesystems: len(p.Mounts), Done: len(p.Done), Visited: p.Visited,
		Found: len(p.Found), Kept: kept, AtLeast: !p.Counted}
	for i, m := range p.Done {
		took := time.Duration(-1)
		if i < len(p.Took) {
			took = p.Took[i]
		}
		sp.Finished = append(sp.Finished, SearchedFilesystem{Mount: m, Took: took})
	}
	for _, m := range p.Mounts {
		if !slices.Contains(p.Done, m) {
			sp.Walking = m
			break
		}
	}
	return sp
}

// Words are the progress in one line, as the verdict says it beside "not judged yet".
func (sp *SearchProgress) Words() string {
	if sp == nil {
		return ""
	}
	s := fmt.Sprintf("progress: %d of %d filesystems walked", sp.Done, sp.Filesystems)
	if sp.Walking != "" {
		s += ", walking " + sp.Walking
	}
	atLeast := ""
	if sp.AtLeast {
		atLeast = "at least "
	}
	s += fmt.Sprintf(", %s%d entries visited, %d setuid- or setgid-root files found so far", atLeast, sp.Visited, sp.Found)
	if sp.Kept.IsZero() {
		s += ", not kept yet"
	} else {
		s += ", kept " + sp.Kept.Local().Format("2006-01-02 15:04:05")
	}
	var took []string
	for _, f := range sp.Finished {
		if f.Took < 0 {
			took = append(took, f.Mount+" in a time not known")
		} else {
			took = append(took, f.Mount+" in "+f.Took.Round(time.Second).String())
		}
	}
	if len(took) > 0 {
		s += "; walked " + strings.Join(took, ", ")
	}
	return s
}

func (e *SetuidNotJudged) Error() string {
	stale := ""
	if !e.Last.IsZero() {
		stale = fmt.Sprintf("; the last complete one, started %s, is too old to judge", e.Last.Local().Format("2006-01-02 15:04"))
	}
	if e.Pending {
		return fmt.Sprintf("%s: the search for setuid programs, started %s, has not finished yet%s", ReasonPending,
			e.Since.Local().Format("2006-01-02 15:04"), stale)
	}
	return fmt.Sprintf("%s: the search for setuid programs did not complete (last tried %s: %v); it is tried "+
		"again later%s", ReasonIncomplete, e.Since.Local().Format("2006-01-02 15:04"), e.Err, stale)
}

// The reasons of a root verdict the setuid search could not answer.
const (
	ReasonPending    = rootsearch.ReasonPending
	ReasonIncomplete = rootsearch.ReasonIncomplete
)

func (c *SetuidCache) every() time.Duration {
	if c.Every > 0 {
		return c.Every
	}
	return rootsearch.Every
}

func (c *SetuidCache) freshFor() time.Duration {
	if c.FreshFor > 0 {
		return c.FreshFor
	}
	return rootsearch.FreshFor
}

// SearchAgain asks for a new search at the next look: an apply changed what root controls (an account, a group,
// a sudo rule, a package, an action). The last complete search keeps judging until it is too old: only root acts
// through an apply, and the agent account cannot make a setuid-root file. Nil c is nothing to ask.
func (c *SetuidCache) SearchAgain() {
	if c == nil {
		return
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	c.again = true
	// The walk under way, or kept, started before root acted: never carried on, even by an engine that
	// restarts before the next search starts (the review of 2026-10-10). It keeps no progress from now on.
	c.unfinished = nil
	c.forgetProgress()
}

// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
// verdict is "not judged yet". Asked at the controller's terminal alone (`nox node setuid-search <node>`): a
// search can never make a machine free wrongly, but asked again and again it would keep a machine unjudged
// and its disks busy. by is who asked, for the journal. Nil c has nothing to throw away.
func (c *SetuidCache) SearchNow(by string) string {
	if c == nil {
		return "this engine keeps no search for setuid programs"
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	c.loaded = true // nothing kept is read after this: it is thrown away
	what := "no complete search was kept"
	if c.last != nil {
		what = "the complete search started " + c.last.Started.Local().Format("2006-01-02 15:04") + " is thrown away"
	}
	if c.running {
		what += ", and the search running since " + c.startedAt.Local().Format("2006-01-02 15:04") + " is ended"
		if c.cancel != nil {
			c.cancel()
		}
	}
	c.gen++
	c.last, c.unfinished, c.progress = nil, nil, nil
	c.failed, c.failedAt, c.backoff, c.again = nil, time.Time{}, 0, false
	c.forgetProgress()
	if c.Path != "" {
		if err := os.Remove(c.Path); err != nil && !errors.Is(err, fs.ErrNotExist) {
			c.say("the search for setuid programs could not forget the complete search it kept: " + err.Error())
		}
	}
	said := "a new search for setuid programs starts at the next look, asked by " + by + ": " + what +
		"; until it completes, whether an account can become root through one is not judged"
	c.say(said)
	return said
}

// SetuidSearch is one search: it carries p on to its end — walking what p has not walked, calling checkpoint
// as it goes so its progress can be kept — and answers the setuid-root programs no package owns.
type SetuidSearch func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error)

// Look answers the setuid-root programs no package owns as the last complete search found them, and when it
// started — while it is fresh — and starts a search in the background when one is due; it never waits for one.
// mounts are the filesystems a new search walks. Nil c searches in place, for a caller that wants to.
//
// **One clock: the caller's.** A search's time is the `now` it was started at, never the wall clock read when it
// finished: due-ness compares a later `now` with it, and mixing the two clocks made a search never due again
// whenever the caller's clock ran behind the wall clock (found while checking issue 356). Measured from the
// start, a result reads at most one search older than it is, which errs towards not judging.
func (c *SetuidCache) Look(now time.Time, mounts []string, search SetuidSearch) ([]string, time.Time, error) {
	if c == nil {
		p := &SetuidProgress{Started: now, Mounts: slices.Clone(mounts), Counted: true}
		found, err := search(context.Background(), p, func() {})
		return found, now, err
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if !c.loaded {
		c.loaded = true
		c.last = c.loadResult()
		c.unfinished = c.loadProgress()
	}
	var age time.Duration
	if c.last != nil {
		age = now.Sub(c.last.Started)
	}
	// Fresh only while it walked every filesystem a search would walk now: one mounted since is not judged
	// by a search that never saw it.
	fresh := c.last != nil && age > -clockSkew && age < c.freshFor() && covers(c.last.Mounts, mounts)
	// Asked for (an apply, a new filesystem): at most once an AgainAfter, counted from when the last search
	// ended, so a walk of hours is not followed by another at once, apply after apply.
	gap := now.Sub(c.attemptAt)
	if !c.endedAt.IsZero() {
		gap = now.Sub(c.endedAt)
	}
	due := !c.running && (c.failed == nil || now.Sub(c.failedAt) >= c.backoff) &&
		(!fresh || age >= c.every() || (c.again && (c.attemptAt.IsZero() || gap >= AgainAfter)))
	if due {
		c.start(now, mounts, search)
	}
	switch {
	case fresh:
		return slices.Clone(c.last.Found), c.last.Started, nil
	case c.running:
		return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: c.startedAt, Last: c.lastStarted(),
			Progress: c.progress}
	case c.failed != nil:
		return nil, time.Time{}, &SetuidNotJudged{Since: c.failedAt, Err: c.failed, Last: c.lastStarted()}
	default:
		// Not fresh, not running, never failed: cannot be, as not fresh is due. Never a verdict.
		return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: now, Last: c.lastStarted()}
	}
}

func (c *SetuidCache) lastStarted() time.Time {
	if c.last == nil {
		return time.Time{}
	}
	return c.last.Started
}

// resumable is the walk that did not finish, when a search may carry it on: younger than FreshFor, and no
// SearchAgain since — an apply that moved root's things wants a walk that starts after it. Anything else is
// thrown away. c.mu held.
func (c *SetuidCache) resumable(now time.Time) *SetuidProgress {
	p := c.unfinished
	c.unfinished = nil
	if p == nil {
		return nil
	}
	age := now.Sub(p.Started)
	if c.again || age <= -clockSkew || age >= c.freshFor() {
		c.forgetProgress()
		return nil
	}
	return p
}

// start runs one search in the background, with no bound: a walk that did not finish carried on, or a new one.
// c.mu held.
func (c *SetuidCache) start(now time.Time, mounts []string, search SetuidSearch) {
	p := c.resumable(now)
	resumed := p != nil
	if p == nil {
		p = &SetuidProgress{ID: fmt.Sprintf("%x", now.UnixNano()), Started: now, Mounts: slices.Clone(mounts),
			Cursor: map[string]string{}, Counted: true}
	}
	c.running, c.startedAt, c.attemptAt, c.again = true, p.Started, now, false
	c.starts++
	// What a look says until the first checkpoint: where the kept walk was, or nothing walked.
	kept := p.Kept
	c.progress = progressOf(p, kept)
	if resumed {
		c.say(fmt.Sprintf("the search for setuid programs started %s carries on where it was (%d of %d "+
			"filesystems walked)", p.Started.Local().Format("2006-01-02 15:04"), len(p.Done), len(p.Mounts)))
	}
	began := time.Now()
	gen := c.gen
	ctx, cancel := context.WithCancel(context.Background())
	c.cancel = cancel
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()
		var found []string
		var err error
		if c.Lower != nil {
			// The goroutine stays on its lowered thread until it ends, and the thread ends with it.
			if lerr := c.Lower(); lerr != nil {
				err = fmt.Errorf("its priority could not be lowered, so it did not walk: %w", lerr)
			}
		}
		if err == nil {
			unkept := false
			found, err = search(ctx, p, func() {
				c.mu.Lock()
				superseded := c.again || c.gen != gen
				c.mu.Unlock()
				if !superseded { // root acted since this walk started: its progress is not to be carried on
					at := time.Now()
					p.Kept = at
					kerr := c.keepProgress(p)
					// Superseded while it wrote (SearchAgain, SearchNow), after they forgot the kept walk: forgotten
					// again here, so no engine restarted before this walk ends carries it on (the review of
					// 2026-10-10). Written outside the lock, so a look never waits on the disk.
					c.mu.Lock()
					if c.again || c.gen != gen {
						c.forgetProgress()
					}
					c.mu.Unlock()
					if kerr != nil {
						p.Kept = kept // not kept: the file still says the time before
						if !unkept {
							unkept = true
							c.say("the search for setuid programs could not keep how far it got, and starts over " +
								"if the engine stops: " + kerr.Error())
						}
					} else {
						kept = at
					}
				}
				// What a look says from now: computed from p here, on the walk's own goroutine, never read
				// by a look while the walk changes it.
				sp := progressOf(p, kept)
				c.mu.Lock()
				if c.gen == gen {
					c.progress = sp
				}
				c.mu.Unlock()
			})
		}
		took := time.Since(began)
		cancel()
		c.mu.Lock()
		defer c.mu.Unlock()
		c.running, c.progress, c.cancel = false, nil, nil
		c.endedAt = now.Add(took)
		if c.gen != gen {
			// Thrown away by SearchNow while it ran: neither a result nor a failure, and nothing to carry on. The
			// next look starts the search asked for.
			c.endedAt = time.Time{}
			// A checkpoint that read its generation just before SearchNow may have kept this walk again.
			c.forgetProgress()
			c.say(fmt.Sprintf("the search for setuid programs started %s was ended, as asked (%s this time)",
				p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second)))
			return
		}
		if err != nil {
			// The back-off counts from when it failed.
			c.failed, c.failedAt = err, c.endedAt
			c.backoff = min(max(2*c.backoff, FailedBackoff), MaxBackoff)
			if !c.again { // superseded by an ask to search again: never carried on
				c.unfinished = p
			}
			c.say(fmt.Sprintf("the search for setuid programs started %s did not complete (%s this time): %v",
				p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), err))
			return
		}
		r := &setuidResult{Started: p.Started, Finished: now.Add(took), Mounts: p.Mounts, Found: found}
		c.last, c.failed, c.backoff = r, nil, 0
		if err := c.keepResult(r); err != nil {
			c.say("the search for setuid programs could not be kept, and judges from memory until the engine " +
				"restarts: " + err.Error())
		}
		c.forgetProgress()
		// The paths in the engine's journal alone: the verdict says one in a home by count.
		named := ""
		if len(found) > 0 {
			named = ": " + strings.Join(found, ", ")
		}
		c.say(fmt.Sprintf("the search for setuid programs started %s completed (%s this time): %d no package "+
			"owns%s", p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), len(found), named))
	}()
}

func (c *SetuidCache) say(s string) {
	if c.Say != nil {
		c.Say(s)
	}
}

// covers says every filesystem a search would walk now was walked.
func covers(walked, now []string) bool {
	for _, m := range now {
		if !slices.Contains(walked, m) {
			return false
		}
	}
	return true
}

// Wait is for a test: until the search running, if any, has finished.
func (c *SetuidCache) Wait() { c.wg.Wait() }

func (c *SetuidCache) keepResult(r *setuidResult) error { return keepFile(c.Path, r) }

func (c *SetuidCache) keepProgress(p *SetuidProgress) error { return keepFile(c.ProgressPath, p) }

// forgetProgress removes the kept walk: finished, or thrown away.
func (c *SetuidCache) forgetProgress() {
	if c.ProgressPath == "" {
		return
	}
	if err := os.Remove(c.ProgressPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
		c.say("the search for setuid programs could not forget the walk it kept: " + err.Error())
	}
}

// keepFile writes v to path: a new file beside it, readable by its owner alone, renamed over it — a link at
// path is replaced, never followed. Empty path keeps nothing.
func keepFile(path string, v any) error {
	if path == "" {
		return nil
	}
	raw, err := json.Marshal(v)
	if err != nil {
		return err
	}
	f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	if _, err := f.Write(raw); err != nil {
		f.Close()
		return err
	}
	if err := f.Sync(); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	return os.Rename(f.Name(), path)
}

// loadResult reads the kept complete search, or nil: none kept, or one that cannot be believed, said why.
func (c *SetuidCache) loadResult() *setuidResult {
	var r setuidResult
	if !c.loadFile(c.Path, "complete search", &r) {
		return nil
	}
	if r.Started.IsZero() || r.Finished.Before(r.Started) || len(r.Mounts) == 0 {
		c.say("the kept search for setuid programs says no complete search, and a new one runs")
		return nil
	}
	return &r
}

// loadProgress reads the kept walk, or nil.
func (c *SetuidCache) loadProgress() *SetuidProgress {
	var p SetuidProgress
	if !c.loadFile(c.ProgressPath, "walk under way", &p) {
		return nil
	}
	if p.Started.IsZero() || len(p.Mounts) == 0 {
		c.say("the kept walk for setuid programs says no walk, and it starts over")
		c.forgetProgress()
		return nil
	}
	if p.Cursor == nil {
		p.Cursor = map[string]string{}
	}
	return &p
}

func (c *SetuidCache) loadFile(path, what string, v any) bool {
	if path == "" {
		return false
	}
	raw, err := keptFile(path)
	if errors.Is(err, fs.ErrNotExist) {
		return false
	}
	if err == nil {
		if err = json.Unmarshal(raw, v); err != nil {
			err = fmt.Errorf("%s is not JSON: %w", path, err)
		}
	}
	if err != nil {
		c.say(fmt.Sprintf("the kept %s for setuid programs is not read, and a new one runs: %v", what, err))
		return false
	}
	return true
}

// keptFile reads a kept file, believing it only as a regular file, not followed through a link, owned by the
// engine's own account and writable by it alone, in a directory that is the same: anything else could have been
// written by somebody other than root, and a search it says is no search.
func keptFile(path string) ([]byte, error) {
	if err := ownedAlone(filepath.Dir(path), true); err != nil {
		return nil, err
	}
	f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	info, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if err := ownedAloneInfo(path, info, false); err != nil {
		return nil, err
	}
	return io.ReadAll(io.LimitReader(f, 64<<20))
}

func ownedAlone(path string, dir bool) error {
	info, err := os.Lstat(path)
	if err != nil {
		return err
	}
	return ownedAloneInfo(path, info, dir)
}

func ownedAloneInfo(path string, info fs.FileInfo, dir bool) error {
	if dir && !info.IsDir() || !dir && !info.Mode().IsRegular() {
		return fmt.Errorf("%s is not a plain %s", path, map[bool]string{true: "directory", false: "file"}[dir])
	}
	st, ok := info.Sys().(*syscall.Stat_t)
	if !ok {
		return fmt.Errorf("%s's owner could not be read", path)
	}
	if int(st.Uid) != os.Geteuid() {
		return fmt.Errorf("%s is owned by uid %d, not the engine's %d", path, st.Uid, os.Geteuid())
	}
	if info.Mode().Perm()&0o022 != 0 {
		return fmt.Errorf("%s may be written by others than its owner (mode %o)", path, info.Mode().Perm())
	}
	return nil
}