package main // A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a // restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed // night behind, so the health also asks how long this machine has been up, with the night's process running, // since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has // missed nothing it could have done, and the night's process runs one soon after it starts again. // // The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while // the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time // the process was not running is not counted either: that is its unit's to say, judged by liveness. import ( "encoding/json" "os" "path/filepath" "time" ) // Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours // a night may take or be late by. const Overdue = 26 * time.Hour // LookEvery is how often the night's process counts; Stale how old its count may be before it is said. const ( LookEvery = time.Minute Stale = 10 * time.Minute ) // Awake is the night's process's count: since which good night (zero before the first), how long the // machine was up with the process running, when the process started and when it last counted. type Awake struct { From time.Time `json:"from"` Up time.Duration `json:"up"` Started time.Time `json:"started"` Looked time.Time `json:"looked"` // Boot and Mono are the machine's boot and monotonic clock when it last counted (machineClock). Boot string `json:"boot,omitempty"` Mono time.Duration `json:"mono,omitempty"` } // age is how long ago the count was made. A laptop's count was last written before it suspended, so by the // wall clock it reads hours old on waking until the next look; within one boot it is judged by the monotonic // clock, which stops while the machine sleeps (novox/hq issue 489). After a restart of the machine that clock // began again, and the wall clock judges. func (a Awake) age(s Seen) time.Duration { if a.Boot != "" && a.Boot == s.Boot && a.Mono > 0 && s.Mono >= a.Mono { return s.Mono - a.Mono } return s.Now.Sub(a.Looked) } func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") } // LastAwake is the count as the night's process last wrote it; nil before it ever did. func (b *Backups) LastAwake() *Awake { raw, err := os.ReadFile(b.awakeFile()) if err != nil { return nil } var a Awake if json.Unmarshal(raw, &a) != nil { return nil } return &a } func (b *Backups) keepAwake(a Awake) { raw, _ := json.MarshalIndent(a, "", " ") if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording how long this machine was up since its last good night failed: %v", err) } } // newestGood is the newest good night of any module; zero before the first. func newestGood(nights map[string]Night) time.Time { var newest time.Time for _, n := range nights { if n.OK && n.At.After(newest) { newest = n.At } } return newest } // Count adds one look: up is how long the machine was awake since the last one. A newer good night starts // the count again. Pure. func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake { if !good.Equal(a.From) { a.From, a.Up = good, 0 } if up > 0 { a.Up += up } a.Looked = now.UTC() return a } // counting is the night's process's count, once a minute while it runs. It carries on from what an earlier // process left, so a restart neither forgives an overdue night nor starts one fresh. func counting(b *Backups) { a := Awake{} if was := b.LastAwake(); was != nil { a = *was } a.Started = time.Now().UTC() last := time.Now() for { now := time.Now() // Both readings carry the monotonic clock, which does not run while the machine is suspended. a = a.Count(newestGood(b.Nights()), now.Sub(last), now) a.Boot, a.Mono = machineClock() last = now b.keepAwake(a) time.Sleep(LookEvery) } }