jschoubben
f311fa36dc
A file's own details (lstat) may be read from the disk as a directory is, so a directory of tens of thousands of files was read without a look and the header's promise that no burst is longer than PaceLook did not hold. The walk now asks the pacer before each file as well; the 250 ms gate makes that a clock read per file. A test with one directory of twenty files fails without it. The clock is read once after a run of pauses, and the constants test says that it guards the constants only.
303 lines
9.4 KiB
Go
303 lines
9.4 KiB
Go
package accounts
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/rootsearch"
|
|
)
|
|
|
|
// The walk for setuid programs yields to a busy disk by the machine's I/O pressure (novox/hq issue 433), because
|
|
// the idle I/O class it is put in is ignored by a disk scheduler that does not schedule by class, as the
|
|
// control-node's mq-deadline does not — and it keeps a tenth of the time whatever the pressure, so it still ends.
|
|
|
|
// fakePace is a pacer whose stall counter, sleeps and clock a test controls.
|
|
type fakePace struct {
|
|
// shares is the stall share, in percent, of each interval between two reads of the counter, in turn; the last
|
|
// is said again once they run out. The first read only sets the counter's start.
|
|
shares []float64
|
|
i int
|
|
total time.Duration
|
|
lastRead time.Time
|
|
reads int
|
|
|
|
clock time.Time
|
|
// tick moves the clock on at every look at it, as a walk reading directories takes time; zero does not.
|
|
tick time.Duration
|
|
slept []time.Duration
|
|
said []string
|
|
fail error
|
|
}
|
|
|
|
func newFake(shares ...float64) *fakePace {
|
|
return &fakePace{shares: shares, clock: time.Unix(1_000_000, 0)}
|
|
}
|
|
|
|
func (f *fakePace) pacer() *pacer {
|
|
return &pacer{
|
|
stalled: func() (time.Duration, error) {
|
|
if f.fail != nil {
|
|
return 0, f.fail
|
|
}
|
|
f.reads++
|
|
if !f.lastRead.IsZero() {
|
|
s := f.shares[min(f.i, len(f.shares)-1)]
|
|
f.i++
|
|
f.total += time.Duration(float64(f.clock.Sub(f.lastRead)) * s / 100)
|
|
}
|
|
f.lastRead = f.clock
|
|
return f.total, nil
|
|
},
|
|
sleep: func(ctx context.Context, d time.Duration) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
f.slept = append(f.slept, d)
|
|
f.clock = f.clock.Add(d)
|
|
return nil
|
|
},
|
|
now: func() time.Time {
|
|
f.clock = f.clock.Add(f.tick)
|
|
return f.clock
|
|
},
|
|
say: func(s string) { f.said = append(f.said, s) },
|
|
}
|
|
}
|
|
|
|
func (f *fakePace) paused() time.Duration {
|
|
var total time.Duration
|
|
for _, d := range f.slept {
|
|
total += d
|
|
}
|
|
return total
|
|
}
|
|
|
|
func TestTheWalkPausesWhileTheDiskIsUnderPressureAndGoesOnWhenItEases(t *testing.T) {
|
|
f := newFake(60, 40, 3)
|
|
pc := f.pacer()
|
|
if err := pc.yield(t.Context()); err != nil { // the first look sets where the counter counts from
|
|
t.Fatal(err)
|
|
}
|
|
f.clock = f.clock.Add(300 * time.Millisecond) // the walk reads
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// 60 % since the first look: pause; 40 % during the first pause: pause again; 3 %: go on.
|
|
if want := []time.Duration{PaceStep, PaceStep}; !slices.Equal(f.slept, want) {
|
|
t.Fatalf("slept %v, want %v", f.slept, want)
|
|
}
|
|
}
|
|
|
|
func TestTheFirstLookOnlySetsWhereTheNextCountsFrom(t *testing.T) {
|
|
f := newFake(99)
|
|
if err := f.pacer().yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(f.slept) != 0 {
|
|
t.Fatalf("slept %v on the first look, which has nothing to compare", f.slept)
|
|
}
|
|
}
|
|
|
|
func TestTheWalkDoesNotPauseOnAQuietDisk(t *testing.T) {
|
|
f := newFake(0.4)
|
|
pc := f.pacer()
|
|
for range 10 {
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.clock = f.clock.Add(PaceLook)
|
|
}
|
|
if len(f.slept) != 0 {
|
|
t.Fatalf("slept %v on a quiet disk", f.slept)
|
|
}
|
|
}
|
|
|
|
// The control-node's own load sits near the bound with no walk running: the walk must still end (issue 361).
|
|
func TestTheWalkKeepsATenthOfTheTimeWhateverThePressure(t *testing.T) {
|
|
f := newFake(100)
|
|
pc := f.pacer()
|
|
var read time.Duration
|
|
for range 400 {
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.clock = f.clock.Add(PaceLook)
|
|
read += PaceLook
|
|
}
|
|
if f.paused() == 0 {
|
|
t.Fatal("never paused under constant pressure")
|
|
}
|
|
if f.paused() > PaceRatio*read {
|
|
t.Fatalf("paused %v while it read %v: more than %d times as long", f.paused(), read, PaceRatio)
|
|
}
|
|
}
|
|
|
|
// What the floor promises against the ages a search is judged by: the walk that took 16m31s unpaced on the
|
|
// control-node on 2026-10-11, slowed to its worst, still completes well inside the time the controller stays quiet
|
|
// while it waits for a search. A guard on the constants only: that the code keeps to the ratio is proven by
|
|
// TestTheWalkKeepsATenthOfTheTimeWhateverThePressure.
|
|
func TestTheSlowestPacedWalkStillJudgesInTime(t *testing.T) {
|
|
const unpaced = 16*time.Minute + 31*time.Second
|
|
worst := (1 + PaceRatio) * unpaced
|
|
if worst >= rootsearch.Quiet {
|
|
t.Fatalf("a walk paced to its worst takes %v, not less than the %v the controller waits quietly", worst,
|
|
rootsearch.Quiet)
|
|
}
|
|
}
|
|
|
|
func TestARunOfPausesIsAtMostPaceAtMost(t *testing.T) {
|
|
f := newFake(100)
|
|
pc := f.pacer()
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.clock = f.clock.Add(10 * time.Minute) // read a long time without a look: the ratio would allow 90 minutes
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.paused() != PaceAtMost {
|
|
t.Fatalf("paused %v in a row, want %v", f.paused(), PaceAtMost)
|
|
}
|
|
}
|
|
|
|
func TestTheWalkLooksAtThePressureAtMostEveryPaceLook(t *testing.T) {
|
|
f := newFake(1)
|
|
pc := f.pacer()
|
|
for range 5 {
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.clock = f.clock.Add(PaceLook / 10)
|
|
}
|
|
if f.reads != 1 {
|
|
t.Fatalf("read the pressure %d times within %v, want once", f.reads, PaceLook)
|
|
}
|
|
f.clock = f.clock.Add(PaceLook)
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.reads != 2 {
|
|
t.Fatalf("read the pressure %d times after %v, want twice", f.reads, PaceLook)
|
|
}
|
|
}
|
|
|
|
// Pressure that cannot be read pauses nothing, and is said once, with what it means, rather than at every look.
|
|
func TestAPressureThatCannotBeReadIsSaidOnce(t *testing.T) {
|
|
f := newFake(0)
|
|
f.fail = errors.New("no such file")
|
|
pc := f.pacer()
|
|
for range 3 {
|
|
if err := pc.yield(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.clock = f.clock.Add(PaceLook)
|
|
}
|
|
if len(f.slept) != 0 || len(f.said) != 1 {
|
|
t.Fatalf("slept %v, said %q; want no pause and one line", f.slept, f.said)
|
|
}
|
|
if !strings.Contains(f.said[0], "full priority") {
|
|
t.Errorf("the line does not say what it means for the disk: %q", f.said[0])
|
|
}
|
|
}
|
|
|
|
// The engine's own walk is paced by the machine's pressure, and says through the search's own channel.
|
|
func TestTheEnginesWalkIsPacedByTheMachinesPressure(t *testing.T) {
|
|
var said []string
|
|
w := engineWalker(nil, func(s string) { said = append(said, s) })
|
|
if w.pace == nil || w.pace.path != ioPressureFile || w.pace.stalled == nil {
|
|
t.Fatalf("the engine's walker is not paced by %s: %+v", ioPressureFile, w.pace)
|
|
}
|
|
w.pace.say("x")
|
|
if len(said) != 1 {
|
|
t.Fatal("the engine's walker does not say through the search's channel")
|
|
}
|
|
}
|
|
|
|
// many is a tree of enough directories that a walk reaches a look at the pressure partway.
|
|
func many(t *testing.T) string {
|
|
names := []string{"bin/", "bin/su!", "bin/ls", "sbin/", "sbin/mount!", "usr/lib/helper%"}
|
|
for _, d := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
|
|
names = append(names, "usr/share/"+d+"/")
|
|
}
|
|
return tree(t, names...)
|
|
}
|
|
|
|
// A walk ended while it pauses is an incomplete walk, never one that found nothing.
|
|
func TestAWalkEndedWhileItPausesIsIncomplete(t *testing.T) {
|
|
top := many(t)
|
|
f := newFake(80)
|
|
f.tick = 100 * time.Millisecond
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
pc := f.pacer()
|
|
// The engine stops while the walk waits for the disk: the context ends inside the pause, not before the walk.
|
|
pc.sleep = func(ctx context.Context, d time.Duration) error {
|
|
f.slept = append(f.slept, d)
|
|
cancel()
|
|
return ctx.Err()
|
|
}
|
|
w := mine()
|
|
w.pace = pc
|
|
err := w.walk(ctx, &SetuidProgress{Mounts: []string{top}}, func() {})
|
|
if len(f.slept) == 0 {
|
|
t.Fatal("the walk never paused, so this proves nothing about a walk ended while it pauses")
|
|
}
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("a walk ended while it paused answered %v", err)
|
|
}
|
|
}
|
|
|
|
// Paced, the walk finds what it finds unpaced: pausing changes when, never what.
|
|
func TestAPacedWalkFindsWhatAnUnpacedWalkFinds(t *testing.T) {
|
|
top := many(t)
|
|
plain := walkAll(t, mine(), top)
|
|
|
|
f := newFake(80, 2)
|
|
f.tick = 100 * time.Millisecond
|
|
w := mine()
|
|
w.pace = f.pacer()
|
|
paced := walkAll(t, w, top)
|
|
if !slices.Equal(plain, paced) {
|
|
t.Fatalf("paced walk found %v, unpaced %v", paced, plain)
|
|
}
|
|
if len(f.slept) == 0 {
|
|
t.Fatal("the walk read its directories without asking the pacer: it never paused under pressure")
|
|
}
|
|
}
|
|
|
|
func TestTheIOPressureIsReadAsTheKernelWritesIt(t *testing.T) {
|
|
got, err := parseIOStalled([]byte("some avg10=26.24 avg60=20.07 avg300=27.47 total=123456789\n" +
|
|
"full avg10=21.90 avg60=15.77 avg300=22.47 total=98765432\n"))
|
|
if err != nil || got != 123456789*time.Microsecond {
|
|
t.Fatalf("read %v, %v; want 123456789 µs", got, err)
|
|
}
|
|
for _, bad := range []string{"", "full avg10=1.00 avg60=0 avg300=0 total=1\n", "some avg10=1 total=x\n"} {
|
|
if _, err := parseIOStalled([]byte(bad)); err == nil {
|
|
t.Errorf("%q was read as a pressure", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One directory of many files is paced as it is read, not only between directories: a file's own details may be
|
|
// read from the disk too.
|
|
func TestADirectoryOfManyFilesIsPacedAsItIsRead(t *testing.T) {
|
|
names := []string{"bin/"}
|
|
for i := range 20 {
|
|
names = append(names, "bin/f"+strings.Repeat("x", i+1))
|
|
}
|
|
top := tree(t, names...)
|
|
f := newFake(80, 2)
|
|
f.tick = 100 * time.Millisecond
|
|
w := mine()
|
|
w.pace = f.pacer()
|
|
walkAll(t, w, top)
|
|
// Two directories alone would be looked at once, 200 ms apart, within one look: no pause at all.
|
|
if len(f.slept) == 0 {
|
|
t.Fatal("a directory of twenty files was read without a pause under pressure")
|
|
}
|
|
}
|