package accounts import ( "context" "errors" "slices" "strings" "testing" "time" "github.com/novox/mesh-host/rootsearch" ) // The walk for setuid programs yields to a busy disk by the machine's I/O pressure (novox/hq issue 433), because // the idle I/O class it is put in is ignored by a disk scheduler that does not schedule by class, as the // control-node's mq-deadline does not — and it keeps a tenth of the time whatever the pressure, so it still ends. // fakePace is a pacer whose stall counter, sleeps and clock a test controls. type fakePace struct { // shares is the stall share, in percent, of each interval between two reads of the counter, in turn; the last // is said again once they run out. The first read only sets the counter's start. shares []float64 i int total time.Duration lastRead time.Time reads int clock time.Time // tick moves the clock on at every look at it, as a walk reading directories takes time; zero does not. tick time.Duration slept []time.Duration said []string fail error } func newFake(shares ...float64) *fakePace { return &fakePace{shares: shares, clock: time.Unix(1_000_000, 0)} } func (f *fakePace) pacer() *pacer { return &pacer{ stalled: func() (time.Duration, error) { if f.fail != nil { return 0, f.fail } f.reads++ if !f.lastRead.IsZero() { s := f.shares[min(f.i, len(f.shares)-1)] f.i++ f.total += time.Duration(float64(f.clock.Sub(f.lastRead)) * s / 100) } f.lastRead = f.clock return f.total, nil }, sleep: func(ctx context.Context, d time.Duration) error { if err := ctx.Err(); err != nil { return err } f.slept = append(f.slept, d) f.clock = f.clock.Add(d) return nil }, now: func() time.Time { f.clock = f.clock.Add(f.tick) return f.clock }, say: func(s string) { f.said = append(f.said, s) }, } } func (f *fakePace) paused() time.Duration { var total time.Duration for _, d := range f.slept { total += d } return total } func TestTheWalkPausesWhileTheDiskIsUnderPressureAndGoesOnWhenItEases(t *testing.T) { f := newFake(60, 40, 3) pc := f.pacer() if err := pc.yield(t.Context()); err != nil { // the first look sets where the counter counts from t.Fatal(err) } f.clock = f.clock.Add(300 * time.Millisecond) // the walk reads if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } // 60 % since the first look: pause; 40 % during the first pause: pause again; 3 %: go on. if want := []time.Duration{PaceStep, PaceStep}; !slices.Equal(f.slept, want) { t.Fatalf("slept %v, want %v", f.slept, want) } } func TestTheFirstLookOnlySetsWhereTheNextCountsFrom(t *testing.T) { f := newFake(99) if err := f.pacer().yield(t.Context()); err != nil { t.Fatal(err) } if len(f.slept) != 0 { t.Fatalf("slept %v on the first look, which has nothing to compare", f.slept) } } func TestTheWalkDoesNotPauseOnAQuietDisk(t *testing.T) { f := newFake(0.4) pc := f.pacer() for range 10 { if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } f.clock = f.clock.Add(PaceLook) } if len(f.slept) != 0 { t.Fatalf("slept %v on a quiet disk", f.slept) } } // The control-node's own load sits near the bound with no walk running: the walk must still end (issue 361). func TestTheWalkKeepsATenthOfTheTimeWhateverThePressure(t *testing.T) { f := newFake(100) pc := f.pacer() var read time.Duration for range 400 { if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } f.clock = f.clock.Add(PaceLook) read += PaceLook } if f.paused() == 0 { t.Fatal("never paused under constant pressure") } if f.paused() > PaceRatio*read { t.Fatalf("paused %v while it read %v: more than %d times as long", f.paused(), read, PaceRatio) } } // What the floor promises against the ages a search is judged by: the walk that took 16m31s unpaced on the // control-node on 2026-10-11, slowed to its worst, still completes well inside the time the controller stays quiet // while it waits for a search. A guard on the constants only: that the code keeps to the ratio is proven by // TestTheWalkKeepsATenthOfTheTimeWhateverThePressure. func TestTheSlowestPacedWalkStillJudgesInTime(t *testing.T) { const unpaced = 16*time.Minute + 31*time.Second worst := (1 + PaceRatio) * unpaced if worst >= rootsearch.Quiet { t.Fatalf("a walk paced to its worst takes %v, not less than the %v the controller waits quietly", worst, rootsearch.Quiet) } } func TestARunOfPausesIsAtMostPaceAtMost(t *testing.T) { f := newFake(100) pc := f.pacer() if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } f.clock = f.clock.Add(10 * time.Minute) // read a long time without a look: the ratio would allow 90 minutes if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } if f.paused() != PaceAtMost { t.Fatalf("paused %v in a row, want %v", f.paused(), PaceAtMost) } } func TestTheWalkLooksAtThePressureAtMostEveryPaceLook(t *testing.T) { f := newFake(1) pc := f.pacer() for range 5 { if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } f.clock = f.clock.Add(PaceLook / 10) } if f.reads != 1 { t.Fatalf("read the pressure %d times within %v, want once", f.reads, PaceLook) } f.clock = f.clock.Add(PaceLook) if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } if f.reads != 2 { t.Fatalf("read the pressure %d times after %v, want twice", f.reads, PaceLook) } } // Pressure that cannot be read pauses nothing, and is said once, with what it means, rather than at every look. func TestAPressureThatCannotBeReadIsSaidOnce(t *testing.T) { f := newFake(0) f.fail = errors.New("no such file") pc := f.pacer() for range 3 { if err := pc.yield(t.Context()); err != nil { t.Fatal(err) } f.clock = f.clock.Add(PaceLook) } if len(f.slept) != 0 || len(f.said) != 1 { t.Fatalf("slept %v, said %q; want no pause and one line", f.slept, f.said) } if !strings.Contains(f.said[0], "full priority") { t.Errorf("the line does not say what it means for the disk: %q", f.said[0]) } } // The engine's own walk is paced by the machine's pressure, and says through the search's own channel. func TestTheEnginesWalkIsPacedByTheMachinesPressure(t *testing.T) { var said []string w := engineWalker(nil, func(s string) { said = append(said, s) }) if w.pace == nil || w.pace.path != ioPressureFile || w.pace.stalled == nil { t.Fatalf("the engine's walker is not paced by %s: %+v", ioPressureFile, w.pace) } w.pace.say("x") if len(said) != 1 { t.Fatal("the engine's walker does not say through the search's channel") } } // many is a tree of enough directories that a walk reaches a look at the pressure partway. func many(t *testing.T) string { names := []string{"bin/", "bin/su!", "bin/ls", "sbin/", "sbin/mount!", "usr/lib/helper%"} for _, d := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} { names = append(names, "usr/share/"+d+"/") } return tree(t, names...) } // A walk ended while it pauses is an incomplete walk, never one that found nothing. func TestAWalkEndedWhileItPausesIsIncomplete(t *testing.T) { top := many(t) f := newFake(80) f.tick = 100 * time.Millisecond ctx, cancel := context.WithCancel(t.Context()) defer cancel() pc := f.pacer() // The engine stops while the walk waits for the disk: the context ends inside the pause, not before the walk. pc.sleep = func(ctx context.Context, d time.Duration) error { f.slept = append(f.slept, d) cancel() return ctx.Err() } w := mine() w.pace = pc err := w.walk(ctx, &SetuidProgress{Mounts: []string{top}}, func() {}) if len(f.slept) == 0 { t.Fatal("the walk never paused, so this proves nothing about a walk ended while it pauses") } if !errors.Is(err, context.Canceled) { t.Fatalf("a walk ended while it paused answered %v", err) } } // Paced, the walk finds what it finds unpaced: pausing changes when, never what. func TestAPacedWalkFindsWhatAnUnpacedWalkFinds(t *testing.T) { top := many(t) plain := walkAll(t, mine(), top) f := newFake(80, 2) f.tick = 100 * time.Millisecond w := mine() w.pace = f.pacer() paced := walkAll(t, w, top) if !slices.Equal(plain, paced) { t.Fatalf("paced walk found %v, unpaced %v", paced, plain) } if len(f.slept) == 0 { t.Fatal("the walk read its directories without asking the pacer: it never paused under pressure") } } func TestTheIOPressureIsReadAsTheKernelWritesIt(t *testing.T) { got, err := parseIOStalled([]byte("some avg10=26.24 avg60=20.07 avg300=27.47 total=123456789\n" + "full avg10=21.90 avg60=15.77 avg300=22.47 total=98765432\n")) if err != nil || got != 123456789*time.Microsecond { t.Fatalf("read %v, %v; want 123456789 µs", got, err) } for _, bad := range []string{"", "full avg10=1.00 avg60=0 avg300=0 total=1\n", "some avg10=1 total=x\n"} { if _, err := parseIOStalled([]byte(bad)); err == nil { t.Errorf("%q was read as a pressure", bad) } } } // One directory of many files is paced as it is read, not only between directories: a file's own details may be // read from the disk too. func TestADirectoryOfManyFilesIsPacedAsItIsRead(t *testing.T) { names := []string{"bin/"} for i := range 20 { names = append(names, "bin/f"+strings.Repeat("x", i+1)) } top := tree(t, names...) f := newFake(80, 2) f.tick = 100 * time.Millisecond w := mine() w.pace = f.pacer() walkAll(t, w, top) // Two directories alone would be looked at once, 200 ms apart, within one look: no pause at all. if len(f.slept) == 0 { t.Fatal("a directory of twenty files was read without a pause under pressure") } }