mesh-host/internal/accounts/setuid_pace_test.go

package accounts

import (
	"context"
	"errors"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/rootsearch"
)

// The walk for setuid programs yields to a busy disk by the machine's I/O pressure (novox/hq issue 433), because
// the idle I/O class it is put in is ignored by a disk scheduler that does not schedule by class, as the
// control-node's mq-deadline does not — and it keeps a tenth of the time whatever the pressure, so it still ends.

// fakePace is a pacer whose stall counter, sleeps and clock a test controls.
type fakePace struct {
	// shares is the stall share, in percent, of each interval between two reads of the counter, in turn; the last
	// is said again once they run out. The first read only sets the counter's start.
	shares   []float64
	i        int
	total    time.Duration
	lastRead time.Time
	reads    int

	clock time.Time
	// tick moves the clock on at every look at it, as a walk reading directories takes time; zero does not.
	tick  time.Duration
	slept []time.Duration
	said  []string
	fail  error
}

func newFake(shares ...float64) *fakePace {
	return &fakePace{shares: shares, clock: time.Unix(1_000_000, 0)}
}

func (f *fakePace) pacer() *pacer {
	return &pacer{
		stalled: func() (time.Duration, error) {
			if f.fail != nil {
				return 0, f.fail
			}
			f.reads++
			if !f.lastRead.IsZero() {
				s := f.shares[min(f.i, len(f.shares)-1)]
				f.i++
				f.total += time.Duration(float64(f.clock.Sub(f.lastRead)) * s / 100)
			}
			f.lastRead = f.clock
			return f.total, nil
		},
		sleep: func(ctx context.Context, d time.Duration) error {
			if err := ctx.Err(); err != nil {
				return err
			}
			f.slept = append(f.slept, d)
			f.clock = f.clock.Add(d)
			return nil
		},
		now: func() time.Time {
			f.clock = f.clock.Add(f.tick)
			return f.clock
		},
		say: func(s string) { f.said = append(f.said, s) },
	}
}

func (f *fakePace) paused() time.Duration {
	var total time.Duration
	for _, d := range f.slept {
		total += d
	}
	return total
}

func TestTheWalkPausesWhileTheDiskIsUnderPressureAndGoesOnWhenItEases(t *testing.T) {
	f := newFake(60, 40, 3)
	pc := f.pacer()
	if err := pc.yield(t.Context()); err != nil { // the first look sets where the counter counts from
		t.Fatal(err)
	}
	f.clock = f.clock.Add(300 * time.Millisecond) // the walk reads
	if err := pc.yield(t.Context()); err != nil {
		t.Fatal(err)
	}
	// 60 % since the first look: pause; 40 % during the first pause: pause again; 3 %: go on.
	if want := []time.Duration{PaceStep, PaceStep}; !slices.Equal(f.slept, want) {
		t.Fatalf("slept %v, want %v", f.slept, want)
	}
}

func TestTheFirstLookOnlySetsWhereTheNextCountsFrom(t *testing.T) {
	f := newFake(99)
	if err := f.pacer().yield(t.Context()); err != nil {
		t.Fatal(err)
	}
	if len(f.slept) != 0 {
		t.Fatalf("slept %v on the first look, which has nothing to compare", f.slept)
	}
}

func TestTheWalkDoesNotPauseOnAQuietDisk(t *testing.T) {
	f := newFake(0.4)
	pc := f.pacer()
	for range 10 {
		if err := pc.yield(t.Context()); err != nil {
			t.Fatal(err)
		}
		f.clock = f.clock.Add(PaceLook)
	}
	if len(f.slept) != 0 {
		t.Fatalf("slept %v on a quiet disk", f.slept)
	}
}

// The control-node's own load sits near the bound with no walk running: the walk must still end (issue 361).
func TestTheWalkKeepsATenthOfTheTimeWhateverThePressure(t *testing.T) {
	f := newFake(100)
	pc := f.pacer()
	var read time.Duration
	for range 400 {
		if err := pc.yield(t.Context()); err != nil {
			t.Fatal(err)
		}
		f.clock = f.clock.Add(PaceLook)
		read += PaceLook
	}
	if f.paused() == 0 {
		t.Fatal("never paused under constant pressure")
	}
	if f.paused() > PaceRatio*read {
		t.Fatalf("paused %v while it read %v: more than %d times as long", f.paused(), read, PaceRatio)
	}
}

// What the floor promises against the ages a search is judged by: the walk that took 16m31s unpaced on the
// control-node on 2026-10-11, slowed to its worst, still completes well inside the time the controller stays quiet
// while it waits for a search, and so judges for most of its fresh age.
func TestTheSlowestPacedWalkStillJudgesInTime(t *testing.T) {
	const unpaced = 16*time.Minute + 31*time.Second
	worst := (1 + PaceRatio) * unpaced
	if worst >= rootsearch.Quiet {
		t.Fatalf("a walk paced to its worst takes %v, not less than the %v the controller waits quietly", worst,
			rootsearch.Quiet)
	}
}

func TestARunOfPausesIsAtMostPaceAtMost(t *testing.T) {
	f := newFake(100)
	pc := f.pacer()
	if err := pc.yield(t.Context()); err != nil {
		t.Fatal(err)
	}
	f.clock = f.clock.Add(10 * time.Minute) // read a long time without a look: the ratio would allow 90 minutes
	if err := pc.yield(t.Context()); err != nil {
		t.Fatal(err)
	}
	if f.paused() != PaceAtMost {
		t.Fatalf("paused %v in a row, want %v", f.paused(), PaceAtMost)
	}
}

func TestTheWalkLooksAtThePressureAtMostEveryPaceLook(t *testing.T) {
	f := newFake(1)
	pc := f.pacer()
	for range 5 {
		if err := pc.yield(t.Context()); err != nil {
			t.Fatal(err)
		}
		f.clock = f.clock.Add(PaceLook / 10)
	}
	if f.reads != 1 {
		t.Fatalf("read the pressure %d times within %v, want once", f.reads, PaceLook)
	}
	f.clock = f.clock.Add(PaceLook)
	if err := pc.yield(t.Context()); err != nil {
		t.Fatal(err)
	}
	if f.reads != 2 {
		t.Fatalf("read the pressure %d times after %v, want twice", f.reads, PaceLook)
	}
}

// Pressure that cannot be read pauses nothing, and is said once, with what it means, rather than at every look.
func TestAPressureThatCannotBeReadIsSaidOnce(t *testing.T) {
	f := newFake(0)
	f.fail = errors.New("no such file")
	pc := f.pacer()
	for range 3 {
		if err := pc.yield(t.Context()); err != nil {
			t.Fatal(err)
		}
		f.clock = f.clock.Add(PaceLook)
	}
	if len(f.slept) != 0 || len(f.said) != 1 {
		t.Fatalf("slept %v, said %q; want no pause and one line", f.slept, f.said)
	}
	if !strings.Contains(f.said[0], "full priority") {
		t.Errorf("the line does not say what it means for the disk: %q", f.said[0])
	}
}

// The engine's own walk is paced by the machine's pressure, and says through the search's own channel.
func TestTheEnginesWalkIsPacedByTheMachinesPressure(t *testing.T) {
	var said []string
	w := engineWalker(nil, func(s string) { said = append(said, s) })
	if w.pace == nil || w.pace.path != ioPressureFile || w.pace.stalled == nil {
		t.Fatalf("the engine's walker is not paced by %s: %+v", ioPressureFile, w.pace)
	}
	w.pace.say("x")
	if len(said) != 1 {
		t.Fatal("the engine's walker does not say through the search's channel")
	}
}

// many is a tree of enough directories that a walk reaches a look at the pressure partway.
func many(t *testing.T) string {
	names := []string{"bin/", "bin/su!", "bin/ls", "sbin/", "sbin/mount!", "usr/lib/helper%"}
	for _, d := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
		names = append(names, "usr/share/"+d+"/")
	}
	return tree(t, names...)
}

// A walk ended while it pauses is an incomplete walk, never one that found nothing.
func TestAWalkEndedWhileItPausesIsIncomplete(t *testing.T) {
	top := many(t)
	f := newFake(80)
	f.tick = 100 * time.Millisecond
	ctx, cancel := context.WithCancel(t.Context())
	defer cancel()
	pc := f.pacer()
	// The engine stops while the walk waits for the disk: the context ends inside the pause, not before the walk.
	pc.sleep = func(ctx context.Context, d time.Duration) error {
		f.slept = append(f.slept, d)
		cancel()
		return ctx.Err()
	}
	w := mine()
	w.pace = pc
	err := w.walk(ctx, &SetuidProgress{Mounts: []string{top}}, func() {})
	if len(f.slept) == 0 {
		t.Fatal("the walk never paused, so this proves nothing about a walk ended while it pauses")
	}
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("a walk ended while it paused answered %v", err)
	}
}

// Paced, the walk finds what it finds unpaced: pausing changes when, never what.
func TestAPacedWalkFindsWhatAnUnpacedWalkFinds(t *testing.T) {
	top := many(t)
	plain := walkAll(t, mine(), top)

	f := newFake(80, 2)
	f.tick = 100 * time.Millisecond
	w := mine()
	w.pace = f.pacer()
	paced := walkAll(t, w, top)
	if !slices.Equal(plain, paced) {
		t.Fatalf("paced walk found %v, unpaced %v", paced, plain)
	}
	if len(f.slept) == 0 {
		t.Fatal("the walk read its directories without asking the pacer: it never paused under pressure")
	}
}

func TestTheIOPressureIsReadAsTheKernelWritesIt(t *testing.T) {
	got, err := parseIOStalled([]byte("some avg10=26.24 avg60=20.07 avg300=27.47 total=123456789\n" +
		"full avg10=21.90 avg60=15.77 avg300=22.47 total=98765432\n"))
	if err != nil || got != 123456789*time.Microsecond {
		t.Fatalf("read %v, %v; want 123456789 µs", got, err)
	}
	for _, bad := range []string{"", "full avg10=1.00 avg60=0 avg300=0 total=1\n", "some avg10=1 total=x\n"} {
		if _, err := parseIOStalled([]byte(bad)); err == nil {
			t.Errorf("%q was read as a pressure", bad)
		}
	}
}