package main // A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a // restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed // night behind, so the health also asks how long this machine has been up, with the night's process running, // since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has // missed nothing it could have done, and the night's process runs one soon after it starts again. // // The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while // the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time // the process was not running is not counted either: that is its unit's to say, judged by liveness. import ( "encoding/json" "os" "path/filepath" "time" ) // Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours // a night may take or be late by. const Overdue = 26 * time.Hour // LookEvery is how often the night's process counts; Stale how old its count may be before it is said. const ( LookEvery = time.Minute Stale = 10 * time.Minute ) // Awake is the night's process's count: since which good night (zero before the first), how long the // machine was up with the process running, when the process started and when it last counted. type Awake struct { From time.Time `json:"from"` Up time.Duration `json:"up"` Started time.Time `json:"started"` Looked time.Time `json:"looked"` } func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") } // LastAwake is the count as the night's process last wrote it; nil before it ever did. func (b *Backups) LastAwake() *Awake { raw, err := os.ReadFile(b.awakeFile()) if err != nil { return nil } var a Awake if json.Unmarshal(raw, &a) != nil { return nil } return &a } func (b *Backups) keepAwake(a Awake) { raw, _ := json.MarshalIndent(a, "", " ") if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording how long this machine was up since its last good night failed: %v", err) } } // newestGood is the newest good night of any module; zero before the first. func newestGood(nights map[string]Night) time.Time { var newest time.Time for _, n := range nights { if n.OK && n.At.After(newest) { newest = n.At } } return newest } // Count adds one look: up is how long the machine was awake since the last one. A newer good night starts // the count again. Pure. func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake { if !good.Equal(a.From) { a.From, a.Up = good, 0 } if up > 0 { a.Up += up } a.Looked = now.UTC() return a } // counting is the night's process's count, once a minute while it runs. It carries on from what an earlier // process left, so a restart neither forgives an overdue night nor starts one fresh. func counting(b *Backups) { a := Awake{} if was := b.LastAwake(); was != nil { a = *was } a.Started = time.Now().UTC() last := time.Now() for { now := time.Now() // Both readings carry the monotonic clock, which does not run while the machine is suspended. a = a.Count(newestGood(b.Nights()), now.Sub(last), now) last = now b.keepAwake(a) time.Sleep(LookEvery) } }