diff --git a/modules/restic/cmd/restic-backups/awake.go b/modules/restic/cmd/restic-backups/awake.go index d11ede4c..02de1c05 100644 --- a/modules/restic/cmd/restic-backups/awake.go +++ b/modules/restic/cmd/restic-backups/awake.go @@ -34,6 +34,20 @@ type Awake struct { Up time.Duration `json:"up"` Started time.Time `json:"started"` Looked time.Time `json:"looked"` + // Boot and Mono are the machine's boot and monotonic clock when it last counted (machineClock). + Boot string `json:"boot,omitempty"` + Mono time.Duration `json:"mono,omitempty"` +} + +// age is how long ago the count was made. A laptop's count was last written before it suspended, so by the +// wall clock it reads hours old on waking until the next look; within one boot it is judged by the monotonic +// clock, which stops while the machine sleeps (novox/hq issue 489). After a restart of the machine that clock +// began again, and the wall clock judges. +func (a Awake) age(s Seen) time.Duration { + if a.Boot != "" && a.Boot == s.Boot && a.Mono > 0 && s.Mono >= a.Mono { + return s.Mono - a.Mono + } + return s.Now.Sub(a.Looked) } func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") } @@ -95,6 +109,7 @@ func counting(b *Backups) { now := time.Now() // Both readings carry the monotonic clock, which does not run while the machine is suspended. a = a.Count(newestGood(b.Nights()), now.Sub(last), now) + a.Boot, a.Mono = machineClock() last = now b.keepAwake(a) time.Sleep(LookEvery) diff --git a/modules/restic/cmd/restic-backups/clock.go b/modules/restic/cmd/restic-backups/clock.go new file mode 100644 index 00000000..d149d0a2 --- /dev/null +++ b/modules/restic/cmd/restic-backups/clock.go @@ -0,0 +1,30 @@ +package main + +// The machine's monotonic clock, shared by its processes: the night's process writes it beside its count and +// the health tool, in another process, reads it again (novox/hq issue 489). Go's own monotonic reading is +// only comparable within one process. + +import ( + "os" + "strings" + "syscall" + "time" + "unsafe" +) + +const bootID = "/proc/sys/kernel/random/boot_id" + +// machineClock is this boot's id and CLOCK_MONOTONIC, which on Linux does not run while the machine is +// suspended and begins again at each boot; "" and 0 when either cannot be read. +func machineClock() (string, time.Duration) { + raw, err := os.ReadFile(bootID) + if err != nil { + return "", 0 + } + var ts syscall.Timespec + const clockMonotonic = 1 + if _, _, e := syscall.Syscall(syscall.SYS_CLOCK_GETTIME, clockMonotonic, uintptr(unsafe.Pointer(&ts)), 0); e != 0 { + return "", 0 + } + return strings.TrimSpace(string(raw)), time.Duration(ts.Nano()) +} diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go index f27300c7..1f6257f2 100644 --- a/modules/restic/cmd/restic-backups/health.go +++ b/modules/restic/cmd/restic-backups/health.go @@ -42,6 +42,10 @@ type Seen struct { // before the process ever counted. Awake *Awake Now time.Time + // Boot and Mono are this machine's boot and monotonic clock now (machineClock): within one boot, the count's + // age is judged by the time the machine was awake (novox/hq issue 489). + Boot string + Mono time.Duration } func hours(d time.Duration) string { @@ -71,11 +75,12 @@ func Judge(s Seen) Health { newest = "the newest good night was at " + clock(a.From) } switch { - case s.Now.Sub(a.Looked) > Stale: + case a.age(s) > Stale: found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+ "night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+ "on this machine.", clock(a.Looked), hours(Stale), newest)) - case a.Up > Overdue: + case a.Up > Overdue && len(s.Declared) > 0: + // Nothing declared here, nothing to back up: no night is overdue (novox/hq issue 489). found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+ "night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+ "the repository, or never starts leaves no failed night behind; read the journal of restic-night "+ @@ -191,8 +196,9 @@ func healthySaid(s Seen, said []string) string { // Health reads what the nights left and the retention as it is now, and judges them. func (b *Backups) Health(ctx context.Context) Health { declared, err := b.Declared() + boot, mono := machineClock() return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), - Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()}) + Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now(), Boot: boot, Mono: mono}) } // letGoSaid is the restore points taken by hand a thinning let go, as the health says them (novox/hq issue 453). diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 8f692301..838aa88e 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -401,3 +401,49 @@ func TestARetentionWithoutHandSaysToPushTheMachine(t *testing.T) { t.Fatalf("hand=0 is %s waiting for %+v", h.State, h.Waits) } } + +// A machine whose modules declare nothing to back up has no night to miss: up for days, it is not said to +// have none happening (novox/hq issue 489). +func TestNothingDeclaredHasNoOverdueNight(t *testing.T) { + now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) + seen := Seen{Retention: InForce{Retention: &TodaysRetention}, Now: now, + Awake: &Awake{Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}} + if h := Judge(seen); h.State != health.StateHealthy || strings.Contains(h.Why, "No night is happening") { + t.Fatalf("nothing declared, up 27 hours: %s: %q", h.State, h.Why) + } +} + +// A laptop that slept: the count was last written before it suspended, hours ago by the wall clock, but only +// seconds ago by the time the machine was awake. It is not stale until the machine was awake that long; after a +// restart of the machine the monotonic clock begins again, and the wall clock judges (novox/hq issue 489). +func TestACountWrittenBeforeASleepIsNotStale(t *testing.T) { + good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 11, 18, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, + Retention: InForce{Retention: &TodaysRetention}, Now: now, + Boot: "b1", Mono: 5*time.Hour + 30*time.Second} + + seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b1", Mono: 5 * time.Hour} + if h := Judge(seen); h.State != health.StateHealthy { + t.Fatalf("counted 30 seconds awake ago, 8 hours asleep: %s: %q", h.State, h.Why) + } + + seen.Awake.Mono = 4 * time.Hour + if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { + t.Fatalf("counted an hour awake ago: %s: %q", h.State, h.Why) + } + + seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b0", Mono: 5 * time.Hour} + if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { + t.Fatalf("counted before the machine restarted, 8 hours ago: %s: %q", h.State, h.Why) + } +} + +// The machine's monotonic clock and its boot read on this machine. +func TestTheMachinesClockReads(t *testing.T) { + boot, mono := machineClock() + if boot == "" || mono <= 0 { + t.Fatalf("boot %q, monotonic %s", boot, mono) + } +}