jschoubben
cbe9fa7a2c
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
A hung night, a lock held for hours or a lock that cannot be taken left no failed night behind, so it stayed healthy. The night's process now counts how long the machine was up since its newest good night, by the monotonic clock so a sleeping laptop is not blamed, and past 26 hours it is raised. The retention file is written 0644 and root's instead of read through sudo: the account reads it and still cannot change it. An unreadable file names its cause, and the healthy answer says only what was seen.
103 lines
3.4 KiB
Go
103 lines
3.4 KiB
Go
package main
|
|
|
|
// A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a
|
|
// restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed
|
|
// night behind, so the health also asks how long this machine has been up, with the night's process running,
|
|
// since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has
|
|
// missed nothing it could have done, and the night's process runs one soon after it starts again.
|
|
//
|
|
// The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while
|
|
// the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time
|
|
// the process was not running is not counted either: that is its unit's to say, judged by liveness.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
)
|
|
|
|
// Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours
|
|
// a night may take or be late by.
|
|
const Overdue = 26 * time.Hour
|
|
|
|
// LookEvery is how often the night's process counts; Stale how old its count may be before it is said.
|
|
const (
|
|
LookEvery = time.Minute
|
|
Stale = 10 * time.Minute
|
|
)
|
|
|
|
// Awake is the night's process's count: since which good night (zero before the first), how long the
|
|
// machine was up with the process running, when the process started and when it last counted.
|
|
type Awake struct {
|
|
From time.Time `json:"from"`
|
|
Up time.Duration `json:"up"`
|
|
Started time.Time `json:"started"`
|
|
Looked time.Time `json:"looked"`
|
|
}
|
|
|
|
func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") }
|
|
|
|
// LastAwake is the count as the night's process last wrote it; nil before it ever did.
|
|
func (b *Backups) LastAwake() *Awake {
|
|
raw, err := os.ReadFile(b.awakeFile())
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var a Awake
|
|
if json.Unmarshal(raw, &a) != nil {
|
|
return nil
|
|
}
|
|
return &a
|
|
}
|
|
|
|
func (b *Backups) keepAwake(a Awake) {
|
|
raw, _ := json.MarshalIndent(a, "", " ")
|
|
if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil {
|
|
b.Say("recording how long this machine was up since its last good night failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// newestGood is the newest good night of any module; zero before the first.
|
|
func newestGood(nights map[string]Night) time.Time {
|
|
var newest time.Time
|
|
for _, n := range nights {
|
|
if n.OK && n.At.After(newest) {
|
|
newest = n.At
|
|
}
|
|
}
|
|
return newest
|
|
}
|
|
|
|
// Count adds one look: up is how long the machine was awake since the last one. A newer good night starts
|
|
// the count again. Pure.
|
|
func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake {
|
|
if !good.Equal(a.From) {
|
|
a.From, a.Up = good, 0
|
|
}
|
|
if up > 0 {
|
|
a.Up += up
|
|
}
|
|
a.Looked = now.UTC()
|
|
return a
|
|
}
|
|
|
|
// counting is the night's process's count, once a minute while it runs. It carries on from what an earlier
|
|
// process left, so a restart neither forgives an overdue night nor starts one fresh.
|
|
func counting(b *Backups) {
|
|
a := Awake{}
|
|
if was := b.LastAwake(); was != nil {
|
|
a = *was
|
|
}
|
|
a.Started = time.Now().UTC()
|
|
last := time.Now()
|
|
for {
|
|
now := time.Now()
|
|
// Both readings carry the monotonic clock, which does not run while the machine is suspended.
|
|
a = a.Count(newestGood(b.Nights()), now.Sub(last), now)
|
|
last = now
|
|
b.keepAwake(a)
|
|
time.Sleep(LookEvery)
|
|
}
|
|
}
|