Walk for setuid programs in the engine and carry an interrupted walk on (hq issue 361)

The control-node's walk takes far longer than the engine runs between
updates, so a search that starts over at every restart never completes.
The walk is the engine's own, in lexical order, so its progress is one
cursor per filesystem, kept every minute beside the engine's state and
carried on after a restart while younger than rootsearch.FreshFor. It runs
on a thread of its own at nice 19 and the idle I/O class.

internal/accounts/setuid_cache.go

@@ -25,36 +25,44 @@ import (
 //     package manager answer for every path found, started less than FreshFor ago. Before the first completes,
 //     or once the last is older, the answer is "not judged yet" (ReasonPending) while a search runs, and "not
 //     judged" (ReasonIncomplete) after one failed. Never "none found" from a search that did not finish.
-//   - **It runs to completion**, once at a time, with no bound: a fresh one when the last complete one is Every
-//     old, when SearchAgain asks (an apply changed what root controls), or when a filesystem appeared that the
-//     last one did not walk. One that fails is tried again after a back-off that doubles from FailedBackoff up
-//     to MaxBackoff.
+//   - **It runs to completion**, once at a time, with no bound, on a thread of its own at idle priority (Lower):
+//     a fresh one when the last complete one is Every old, when SearchAgain asks (an apply changed what root
+//     controls), or when a filesystem appeared that the last one did not walk. One that fails is tried again
+//     after a back-off that doubles from FailedBackoff up to MaxBackoff.
+//   - **Resumable**: the walk's progress is kept in ProgressPath every CheckpointEvery, and a search started
+//     again — after an engine restart, an update, a reboot or a failure — carries on from it while it is
+//     younger than FreshFor. A search SearchAgain asked for never resumes an older walk: it starts from nothing.
 //   - **Kept** in Path, beside the engine's other state, with when it started and finished: an engine restart
 //     serves the last complete search rather than judging nothing until a new walk ends.
 //
-// Why a complete search stays sound for a day and more is said in the rootsearch package: only root makes a
-// setuid-root file, never the agent account.
+// Why a complete search stays sound for a day and more, and a resumed walk is sound, is said in the rootsearch
+// package and setuid_walk.go: only root makes a setuid-root file, never the agent account.
 type SetuidCache struct {
 	// Every and FreshFor are rootsearch's when zero.
 	Every, FreshFor time.Duration
-	// Path is the file the last complete search is kept in; empty keeps it in memory only. The engine's own
-	// state directory, which only root writes: a file there that is not a regular file root owns and alone
-	// may write is not read (keptSearch).
-	Path string
-	// Say is where a search's end and a kept search refused are said; nil says nothing.
+	// Path is the file the last complete search is kept in, and ProgressPath the walk under way; empty keeps
+	// either in memory only. The engine's own state directory, which only root writes: a file there that is
+	// not a regular file the engine's account owns and alone may write is not read (keptFile).
+	Path, ProgressPath string
+	// Lower is called on the search's own goroutine before it walks (LowerThisThread on a serving engine); a
+	// priority it cannot lower fails the search rather than walk the disks at full priority. Nil lowers nothing.
+	Lower func() error
+	// Say is where a search's end and a kept file refused are said; nil says nothing.
 	Say func(string)
 
-	mu        sync.Mutex
-	wg        sync.WaitGroup
-	loaded    bool
-	last      *setuidResult // the last complete search
-	running   bool
-	startedAt time.Time // when the search running, or the last one, was started
-	again     bool      // asked to search again since the running or last one started
-	failedAt  time.Time
-	failed    error
-	backoff   time.Duration
-	starts    int // how many searches were started, for a test
+	mu         sync.Mutex
+	wg         sync.WaitGroup
+	loaded     bool
+	last       *setuidResult   // the last complete search
+	unfinished *SetuidProgress // a walk that did not finish, to resume
+	running    bool
+	startedAt  time.Time // when the search running, or the last one, was started (a resumed one: its start)
+	attemptAt  time.Time // when the running or last search was started or resumed
+	again      bool      // asked to search again since the running or last one started
+	failedAt   time.Time
+	failed     error
+	backoff    time.Duration
+	starts     int // how many searches were started or resumed, for a test
 }
 
 // setuidResult is one complete search, as kept.
@@ -77,8 +85,10 @@ const (
 	// clockSkew is how far in the future a kept search may say it started and still judge: a clock set back
 	// further makes it no verdict, never a fresh one.
 	clockSkew = 5 * time.Minute
-	// SetuidSearchFile is the file's name beside the engine's state.
-	SetuidSearchFile = "setuid-search.json"
+	// SetuidSearchFile and SetuidProgressFile are the files' names beside the engine's state: the last
+	// complete search, and the walk under way.
+	SetuidSearchFile   = "setuid-search.json"
+	SetuidProgressFile = "setuid-search-progress.json"
 )
 
 // SetuidNotJudged is the search's answer when no complete search judges: none has finished yet, the last one is
@@ -137,25 +147,30 @@ func (c *SetuidCache) SearchAgain() {
 	c.again = true
 }
 
+// SetuidSearch is one search: it carries p on to its end — walking what p has not walked, calling checkpoint
+// as it goes so its progress can be kept — and answers the setuid-root programs no package owns.
+type SetuidSearch func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error)
+
 // Look answers the setuid-root programs no package owns as the last complete search found them, and when it
 // started — while it is fresh — and starts a search in the background when one is due; it never waits for one.
-// mounts are the filesystems a search walks now. Nil c searches in place, for a caller that wants to.
+// mounts are the filesystems a new search walks. Nil c searches in place, for a caller that wants to.
 //
 // **One clock: the caller's.** A search's time is the `now` it was started at, never the wall clock read when it
 // finished: due-ness compares a later `now` with it, and mixing the two clocks made a search never due again
 // whenever the caller's clock ran behind the wall clock (found while checking issue 356). Measured from the
 // start, a result reads at most one search older than it is, which errs towards not judging.
-func (c *SetuidCache) Look(now time.Time, mounts []string,
-	search func(ctx context.Context, mounts []string) ([]string, error)) ([]string, time.Time, error) {
+func (c *SetuidCache) Look(now time.Time, mounts []string, search SetuidSearch) ([]string, time.Time, error) {
 	if c == nil {
-		found, err := search(context.Background(), mounts)
+		p := &SetuidProgress{Started: now, Mounts: slices.Clone(mounts)}
+		found, err := search(context.Background(), p, func() {})
 		return found, now, err
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if !c.loaded {
 		c.loaded = true
-		c.last = c.load()
+		c.last = c.loadResult()
+		c.unfinished = c.loadProgress()
 	}
 	var age time.Duration
 	if c.last != nil {
@@ -164,7 +179,7 @@ func (c *SetuidCache) Look(now time.Time, mounts []string,
 	fresh := c.last != nil && age > -clockSkew && age < c.freshFor()
 	asked := c.again || (c.last != nil && !covers(c.last.Mounts, mounts))
 	due := !c.running && (c.failed == nil || now.Sub(c.failedAt) >= c.backoff) &&
-		(!fresh || age >= c.every() || (asked && (c.startedAt.IsZero() || now.Sub(c.startedAt) >= AgainAfter)))
+		(!fresh || age >= c.every() || (asked && (c.attemptAt.IsZero() || now.Sub(c.attemptAt) >= AgainAfter)))
 	if due {
 		c.start(now, mounts, search)
 	}
@@ -188,16 +203,60 @@ func (c *SetuidCache) lastStarted() time.Time {
 	return c.last.Started
 }
 
-// start runs one search in the background, with no bound: c.mu held.
-func (c *SetuidCache) start(now time.Time, mounts []string, search func(context.Context, []string) ([]string, error)) {
-	c.running, c.startedAt, c.again = true, now, false
+// resumable is the walk that did not finish, when a search may carry it on: younger than FreshFor, and no
+// SearchAgain since — an apply that moved root's things wants a walk that starts after it. Anything else is
+// thrown away. c.mu held.
+func (c *SetuidCache) resumable(now time.Time) *SetuidProgress {
+	p := c.unfinished
+	c.unfinished = nil
+	if p == nil {
+		return nil
+	}
+	age := now.Sub(p.Started)
+	if c.again || age <= -clockSkew || age >= c.freshFor() {
+		c.forgetProgress()
+		return nil
+	}
+	return p
+}
+
+// start runs one search in the background, with no bound: a walk that did not finish carried on, or a new one.
+// c.mu held.
+func (c *SetuidCache) start(now time.Time, mounts []string, search SetuidSearch) {
+	p := c.resumable(now)
+	resumed := p != nil
+	if p == nil {
+		p = &SetuidProgress{ID: fmt.Sprintf("%x", now.UnixNano()), Started: now, Mounts: slices.Clone(mounts),
+			Cursor: map[string]string{}}
+	}
+	c.running, c.startedAt, c.attemptAt, c.again = true, p.Started, now, false
 	c.starts++
-	mounts = slices.Clone(mounts)
+	if resumed {
+		c.say(fmt.Sprintf("the search for setuid programs started %s carries on where it was (%d of %d "+
+			"filesystems walked)", p.Started.Local().Format("2006-01-02 15:04"), len(p.Done), len(p.Mounts)))
+	}
 	began := time.Now()
 	c.wg.Add(1)
 	go func() {
 		defer c.wg.Done()
-		found, err := search(context.Background(), mounts)
+		var found []string
+		var err error
+		if c.Lower != nil {
+			// The goroutine stays on its lowered thread until it ends, and the thread ends with it.
+			if lerr := c.Lower(); lerr != nil {
+				err = fmt.Errorf("its priority could not be lowered, so it did not walk: %w", lerr)
+			}
+		}
+		if err == nil {
+			unkept := false
+			found, err = search(context.Background(), p, func() {
+				if kerr := c.keepProgress(p); kerr != nil && !unkept {
+					unkept = true
+					c.say("the search for setuid programs could not keep how far it got, and starts over if " +
+						"the engine stops: " + kerr.Error())
+				}
+			})
+		}
 		took := time.Since(began)
 		c.mu.Lock()
 		defer c.mu.Unlock()
@@ -205,18 +264,20 @@ func (c *SetuidCache) start(now time.Time, mounts []string, search func(context.
 		if err != nil {
 			c.failed, c.failedAt = err, now
 			c.backoff = min(max(2*c.backoff, FailedBackoff), MaxBackoff)
-			c.say(fmt.Sprintf("the search for setuid programs started %s did not complete after %s: %v",
-				now.Local().Format("15:04"), took.Round(time.Second), err))
+			c.unfinished = p
+			c.say(fmt.Sprintf("the search for setuid programs started %s did not complete (%s this time): %v",
+				p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), err))
 			return
 		}
-		r := &setuidResult{Started: now, Finished: now.Add(took), Mounts: mounts, Found: found}
+		r := &setuidResult{Started: p.Started, Finished: now.Add(took), Mounts: p.Mounts, Found: found}
 		c.last, c.failed, c.backoff = r, nil, 0
-		if err := c.keep(r); err != nil {
+		if err := c.keepResult(r); err != nil {
 			c.say("the search for setuid programs could not be kept, and judges from memory until the engine " +
 				"restarts: " + err.Error())
 		}
-		c.say(fmt.Sprintf("the search for setuid programs started %s completed in %s: %d no package owns",
-			now.Local().Format("15:04"), took.Round(time.Second), len(found)))
+		c.forgetProgress()
+		c.say(fmt.Sprintf("the search for setuid programs started %s completed (%s this time): %d no package "+
+			"owns", p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), len(found)))
 	}()
 }
 
@@ -239,16 +300,31 @@ func covers(walked, now []string) bool {
 // Wait is for a test: until the search running, if any, has finished.
 func (c *SetuidCache) Wait() { c.wg.Wait() }
 
-// keep writes a complete search to Path: a new file beside it, readable by root alone, renamed over it.
-func (c *SetuidCache) keep(r *setuidResult) error {
-	if c.Path == "" {
+func (c *SetuidCache) keepResult(r *setuidResult) error { return keepFile(c.Path, r) }
+
+func (c *SetuidCache) keepProgress(p *SetuidProgress) error { return keepFile(c.ProgressPath, p) }
+
+// forgetProgress removes the kept walk: finished, or thrown away.
+func (c *SetuidCache) forgetProgress() {
+	if c.ProgressPath == "" {
+		return
+	}
+	if err := os.Remove(c.ProgressPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		c.say("the search for setuid programs could not forget the walk it kept: " + err.Error())
+	}
+}
+
+// keepFile writes v to path: a new file beside it, readable by its owner alone, renamed over it — a link at
+// path is replaced, never followed. Empty path keeps nothing.
+func keepFile(path string, v any) error {
+	if path == "" {
 		return nil
 	}
-	raw, err := json.Marshal(r)
+	raw, err := json.Marshal(v)
 	if err != nil {
 		return err
 	}
-	f, err := os.CreateTemp(filepath.Dir(c.Path), "."+filepath.Base(c.Path)+".*")
+	f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
 	if err != nil {
 		return err
 	}
@@ -264,25 +340,63 @@ func (c *SetuidCache) keep(r *setuidResult) error {
 	if err := f.Close(); err != nil {
 		return err
 	}
-	return os.Rename(f.Name(), c.Path)
+	return os.Rename(f.Name(), path)
 }
 
-// load reads the kept search, or nil: none kept, or one that cannot be believed, said why.
-func (c *SetuidCache) load() *setuidResult {
-	if c.Path == "" {
+// loadResult reads the kept complete search, or nil: none kept, or one that cannot be believed, said why.
+func (c *SetuidCache) loadResult() *setuidResult {
+	var r setuidResult
+	if !c.loadFile(c.Path, "complete search", &r) {
 		return nil
 	}
-	r, err := keptSearch(c.Path)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		c.say("the kept search for setuid programs is not read, and a new one runs: " + err.Error())
+	if r.Started.IsZero() || r.Finished.Before(r.Started) || len(r.Mounts) == 0 {
+		c.say("the kept search for setuid programs says no complete search, and a new one runs")
+		return nil
 	}
-	return r
+	return &r
 }
 
-// keptSearch reads a kept search, believing it only as a regular file, not followed through a link, owned by
-// the engine's own account and writable by it alone, whose directory is the same: anything else could have been
+// loadProgress reads the kept walk, or nil.
+func (c *SetuidCache) loadProgress() *SetuidProgress {
+	var p SetuidProgress
+	if !c.loadFile(c.ProgressPath, "walk under way", &p) {
+		return nil
+	}
+	if p.Started.IsZero() || len(p.Mounts) == 0 {
+		c.say("the kept walk for setuid programs says no walk, and it starts over")
+		c.forgetProgress()
+		return nil
+	}
+	if p.Cursor == nil {
+		p.Cursor = map[string]string{}
+	}
+	return &p
+}
+
+func (c *SetuidCache) loadFile(path, what string, v any) bool {
+	if path == "" {
+		return false
+	}
+	raw, err := keptFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return false
+	}
+	if err == nil {
+		if err = json.Unmarshal(raw, v); err != nil {
+			err = fmt.Errorf("%s is not JSON: %w", path, err)
+		}
+	}
+	if err != nil {
+		c.say(fmt.Sprintf("the kept %s for setuid programs is not read, and a new one runs: %v", what, err))
+		return false
+	}
+	return true
+}
+
+// keptFile reads a kept file, believing it only as a regular file, not followed through a link, owned by the
+// engine's own account and writable by it alone, in a directory that is the same: anything else could have been
 // written by somebody other than root, and a search it says is no search.
-func keptSearch(path string) (*setuidResult, error) {
+func keptFile(path string) ([]byte, error) {
 	if err := ownedAlone(filepath.Dir(path), true); err != nil {
 		return nil, err
 	}
@@ -298,18 +412,7 @@ func keptSearch(path string) (*setuidResult, error) {
 	if err := ownedAloneInfo(path, info, false); err != nil {
 		return nil, err
 	}
-	raw, err := io.ReadAll(io.LimitReader(f, 1<<20))
-	if err != nil {
-		return nil, err
-	}
-	var r setuidResult
-	if err := json.Unmarshal(raw, &r); err != nil {
-		return nil, fmt.Errorf("%s is not a kept search: %w", path, err)
-	}
-	if r.Started.IsZero() || r.Finished.Before(r.Started) || len(r.Mounts) == 0 {
-		return nil, fmt.Errorf("%s says no complete search", path)
-	}
-	return &r, nil
+	return io.ReadAll(io.LimitReader(f, 64<<20))
 }
 
 func ownedAlone(path string, dir bool) error {