restic: judge no night overdue where nothing is declared, and a count's age by the time the machine was awake
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered

A machine that backs up nothing read as missing its night after 26 hours up, and a laptop's count, last
written before it slept, read as stale on waking until the next look (novox/hq issue 489).
This commit is contained in:
2026-10-11 21:10:11 +02:00
parent fca2f92216
commit 7f05fad58a
4 changed files with 100 additions and 3 deletions
@@ -34,6 +34,20 @@ type Awake struct {
Up time.Duration `json:"up"`
Started time.Time `json:"started"`
Looked time.Time `json:"looked"`
// Boot and Mono are the machine's boot and monotonic clock when it last counted (machineClock).
Boot string `json:"boot,omitempty"`
Mono time.Duration `json:"mono,omitempty"`
}
// age is how long ago the count was made. A laptop's count was last written before it suspended, so by the
// wall clock it reads hours old on waking until the next look; within one boot it is judged by the monotonic
// clock, which stops while the machine sleeps (novox/hq issue 489). After a restart of the machine that clock
// began again, and the wall clock judges.
func (a Awake) age(s Seen) time.Duration {
if a.Boot != "" && a.Boot == s.Boot && a.Mono > 0 && s.Mono >= a.Mono {
return s.Mono - a.Mono
}
return s.Now.Sub(a.Looked)
}
func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") }
@@ -95,6 +109,7 @@ func counting(b *Backups) {
now := time.Now()
// Both readings carry the monotonic clock, which does not run while the machine is suspended.
a = a.Count(newestGood(b.Nights()), now.Sub(last), now)
a.Boot, a.Mono = machineClock()
last = now
b.keepAwake(a)
time.Sleep(LookEvery)
@@ -0,0 +1,30 @@
package main
// The machine's monotonic clock, shared by its processes: the night's process writes it beside its count and
// the health tool, in another process, reads it again (novox/hq issue 489). Go's own monotonic reading is
// only comparable within one process.
import (
"os"
"strings"
"syscall"
"time"
"unsafe"
)
const bootID = "/proc/sys/kernel/random/boot_id"
// machineClock is this boot's id and CLOCK_MONOTONIC, which on Linux does not run while the machine is
// suspended and begins again at each boot; "" and 0 when either cannot be read.
func machineClock() (string, time.Duration) {
raw, err := os.ReadFile(bootID)
if err != nil {
return "", 0
}
var ts syscall.Timespec
const clockMonotonic = 1
if _, _, e := syscall.Syscall(syscall.SYS_CLOCK_GETTIME, clockMonotonic, uintptr(unsafe.Pointer(&ts)), 0); e != 0 {
return "", 0
}
return strings.TrimSpace(string(raw)), time.Duration(ts.Nano())
}
+9 -3
View File
@@ -42,6 +42,10 @@ type Seen struct {
// before the process ever counted.
Awake *Awake
Now time.Time
// Boot and Mono are this machine's boot and monotonic clock now (machineClock): within one boot, the count's
// age is judged by the time the machine was awake (novox/hq issue 489).
Boot string
Mono time.Duration
}
func hours(d time.Duration) string {
@@ -71,11 +75,12 @@ func Judge(s Seen) Health {
newest = "the newest good night was at " + clock(a.From)
}
switch {
case s.Now.Sub(a.Looked) > Stale:
case a.age(s) > Stale:
found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+
"night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+
"on this machine.", clock(a.Looked), hours(Stale), newest))
case a.Up > Overdue:
case a.Up > Overdue && len(s.Declared) > 0:
// Nothing declared here, nothing to back up: no night is overdue (novox/hq issue 489).
found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+
"night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+
"the repository, or never starts leaves no failed night behind; read the journal of restic-night "+
@@ -191,8 +196,9 @@ func healthySaid(s Seen, said []string) string {
// Health reads what the nights left and the retention as it is now, and judges them.
func (b *Backups) Health(ctx context.Context) Health {
declared, err := b.Declared()
boot, mono := machineClock()
return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(),
Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()})
Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now(), Boot: boot, Mono: mono})
}
// letGoSaid is the restore points taken by hand a thinning let go, as the health says them (novox/hq issue 453).
@@ -401,3 +401,49 @@ func TestARetentionWithoutHandSaysToPushTheMachine(t *testing.T) {
t.Fatalf("hand=0 is %s waiting for %+v", h.State, h.Waits)
}
}
// A machine whose modules declare nothing to back up has no night to miss: up for days, it is not said to
// have none happening (novox/hq issue 489).
func TestNothingDeclaredHasNoOverdueNight(t *testing.T) {
now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC)
seen := Seen{Retention: InForce{Retention: &TodaysRetention}, Now: now,
Awake: &Awake{Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}}
if h := Judge(seen); h.State != health.StateHealthy || strings.Contains(h.Why, "No night is happening") {
t.Fatalf("nothing declared, up 27 hours: %s: %q", h.State, h.Why)
}
}
// A laptop that slept: the count was last written before it suspended, hours ago by the wall clock, but only
// seconds ago by the time the machine was awake. It is not stale until the machine was awake that long; after a
// restart of the machine the monotonic clock begins again, and the wall clock judges (novox/hq issue 489).
func TestACountWrittenBeforeASleepIsNotStale(t *testing.T) {
good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC)
now := time.Date(2026, 10, 11, 18, 0, 0, 0, time.UTC)
seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
Nights: map[string]Night{"pg": {OK: true, At: good}},
Retention: InForce{Retention: &TodaysRetention}, Now: now,
Boot: "b1", Mono: 5*time.Hour + 30*time.Second}
seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b1", Mono: 5 * time.Hour}
if h := Judge(seen); h.State != health.StateHealthy {
t.Fatalf("counted 30 seconds awake ago, 8 hours asleep: %s: %q", h.State, h.Why)
}
seen.Awake.Mono = 4 * time.Hour
if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
t.Fatalf("counted an hour awake ago: %s: %q", h.State, h.Why)
}
seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b0", Mono: 5 * time.Hour}
if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
t.Fatalf("counted before the machine restarted, 8 hours ago: %s: %q", h.State, h.Why)
}
}
// The machine's monotonic clock and its boot read on this machine.
func TestTheMachinesClockReads(t *testing.T) {
boot, mono := machineClock()
if boot == "" || mono <= 0 {
t.Fatalf("boot %q, monotonic %s", boot, mono)
}
}