Judge the daily report by wall time, and keep one filesystem check at a time (issue 160)

Go's monotonic clock stops while a laptop is suspended, so a day judged by it could pass a day and a half
of wall time before the machine said what it is. And a hung statfs left a goroutine and thread blocked per
apply; now one check runs at a time and reports after it say the sizes unknown until it ends.

cmd/mesh-host/machine_refresh_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -62,3 +63,45 @@ func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) {
 		t.Error("a refusal was due for what the machine is made of")
 	}
 }
+
+// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic
+// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses
+// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first
+// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no
+// monotonic reading, and judge the day as the wall clock says it.
+func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) {
+	before := time.Now() // carries a monotonic reading
+	asleep := before.Add(3 * time.Second)
+	if asleep.Sub(before) != 3*time.Second {
+		t.Fatal("the monotonic reading is not what this test simulates with")
+	}
+	calls := 0
+	w := &adoptionWatch{now: func() time.Time {
+		calls++
+		if calls == 1 {
+			return before
+		}
+		// The wall clock a day on, as a machine's clock reads after waking.
+		return asleep.Round(0).Add(24 * time.Hour)
+	}}
+	report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}}
+	w.said(report)
+	if strings.Contains(w.machineSaid.String(), "m=") {
+		t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid)
+	}
+	if !w.news(report) {
+		t.Error("a day of wall time on, a laptop that slept did not say what it is")
+	}
+}
+
+// And the watch's own clock never carries a monotonic reading.
+func TestTheWatchClockIsWallTime(t *testing.T) {
+	w := &adoptionWatch{}
+	if s := w.clock().String(); strings.Contains(s, "m=") {
+		t.Errorf("the watch's clock carries a monotonic reading: %s", s)
+	}
+	w = &adoptionWatch{now: time.Now}
+	if s := w.clock().String(); strings.Contains(s, "m=") {
+		t.Errorf("a given clock's time kept its monotonic reading: %s", s)
+	}
+}

cmd/mesh-host/main.go

@@ -1364,11 +1364,14 @@ type adoptionWatch struct {
 // inside the day rather than just past it.
 const machineEvery = 23 * time.Hour
 
+// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's
+// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept
+// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day.
 func (w *adoptionWatch) clock() time.Time {
 	if w.now != nil {
-		return w.now()
+		return w.now().Round(0)
 	}
-	return time.Now()
+	return time.Now().Round(0)
 }
 
 // machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago

internal/profile/machine.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -362,6 +363,9 @@ var localTypes = map[string]bool{"ext2": true, "ext3": true, "ext4": true, "xfs"
 	"zfs": true, "f2fs": true, "vfat": true, "exfat": true, "ntfs": true, "ntfs3": true, "fuseblk": true,
 	"jfs": true, "reiserfs": true, "bcachefs": true}
 
+// sizing is held while a check of the filesystems' sizes runs.
+var sizing sync.Mutex
+
 // filesystemsOf lists each local filesystem once — a device mounted at several places (a btrfs
 // subvolume, a bind mount) under its shortest mount — with its size and what is free. Bounded by the
 // context, so a filesystem whose statfs hangs cannot hang the apply that reports it.
@@ -385,12 +389,20 @@ func filesystemsOf(ctx context.Context, mounts string, stat func(string) (int64,
 		return nil, "/proc/self/mounts lists no local filesystem"
 	}
 
+	// One check at a time, across every report: a filesystem whose statfs hangs keeps the one check that
+	// asked it, and every report after it says the sizes unknown until it finishes, rather than leaving
+	// another goroutine and thread blocked behind it on every apply.
+	if !sizing.TryLock() {
+		return nil, "the last check of the filesystems' sizes has not finished yet"
+	}
+
 	type answer struct {
 		fs  []Filesystem
 		why []string
 	}
 	done := make(chan answer, 1)
 	go func() {
+		defer sizing.Unlock()
 		var a answer
 		for _, fs := range byDevice {
 			size, free, err := stat(fs.Mount)

internal/profile/machine_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 )
@@ -112,14 +113,37 @@ func TestMachine_aFactItCannotReadIsSaidUnknownNeverZero(t *testing.T) {
 func TestMachine_aHungFilesystemCannotHangTheReport(t *testing.T) {
 	src := given(files)
 	block := make(chan struct{})
-	defer close(block)
-	src.Statfs = func(string) (int64, int64, error) { <-block; return 0, 0, nil }
+	var asked atomic.Int32
+	src.Statfs = func(string) (int64, int64, error) { asked.Add(1); <-block; return 0, 0, nil }
+	defer func() {
+		close(block)
+		// The check that hung ends once its filesystem answers, and lets the next one run.
+		for !sizing.TryLock() {
+			time.Sleep(time.Millisecond)
+		}
+		sizing.Unlock()
+	}()
 	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
 	defer cancel()
 	m := DescribeMachine(ctx, src)
 	if len(m.Filesystems) != 0 || !strings.Contains(m.Unknown[FactFilesystems], "did not finish") {
 		t.Errorf("filesystems %v, unknown %q", m.Filesystems, m.Unknown[FactFilesystems])
 	}
+	stuck := asked.Load()
+
+	// Every report while it hangs says so at once, and asks no filesystem again: no goroutine is left
+	// blocked behind the hung one per apply.
+	for i := 0; i < 3; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+		m = DescribeMachine(ctx, src)
+		cancel()
+		if !strings.Contains(m.Unknown[FactFilesystems], "has not finished yet") {
+			t.Errorf("a report while the check hangs said %q", m.Unknown[FactFilesystems])
+		}
+	}
+	if asked.Load() != stuck {
+		t.Errorf("filesystems asked %d times while one check hung, was %d", asked.Load(), stuck)
+	}
 }
 
 func TestMachine_coresUnsaidAreUnknownNotGuessed(t *testing.T) {