mesh-catalog/modules/restic/cmd/restic-backups/awake.go

package main

// A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a
// restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed
// night behind, so the health also asks how long this machine has been up, with the night's process running,
// since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has
// missed nothing it could have done, and the night's process runs one soon after it starts again.
//
// The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while
// the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time
// the process was not running is not counted either: that is its unit's to say, judged by liveness.

import (
	"encoding/json"
	"os"
	"path/filepath"
	"time"
)

// Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours
// a night may take or be late by.
const Overdue = 26 * time.Hour

// LookEvery is how often the night's process counts; Stale how old its count may be before it is said.
const (
	LookEvery = time.Minute
	Stale     = 10 * time.Minute
)

// Awake is the night's process's count: since which good night (zero before the first), how long the
// machine was up with the process running, when the process started and when it last counted.
type Awake struct {
	From    time.Time     `json:"from"`
	Up      time.Duration `json:"up"`
	Started time.Time     `json:"started"`
	Looked  time.Time     `json:"looked"`
	// Boot and Mono are the machine's boot and monotonic clock when it last counted (machineClock).
	Boot string        `json:"boot,omitempty"`
	Mono time.Duration `json:"mono,omitempty"`
}

// age is how long ago the count was made. A laptop's count was last written before it suspended, so by the
// wall clock it reads hours old on waking until the next look; within one boot it is judged by the monotonic
// clock, which stops while the machine sleeps (novox/hq issue 489). After a restart of the machine that clock
// began again, and the wall clock judges.
func (a Awake) age(s Seen) time.Duration {
	if a.Boot != "" && a.Boot == s.Boot && a.Mono > 0 && s.Mono >= a.Mono {
		return s.Mono - a.Mono
	}
	return s.Now.Sub(a.Looked)
}

func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") }

// LastAwake is the count as the night's process last wrote it; nil before it ever did.
func (b *Backups) LastAwake() *Awake {
	raw, err := os.ReadFile(b.awakeFile())
	if err != nil {
		return nil
	}
	var a Awake
	if json.Unmarshal(raw, &a) != nil {
		return nil
	}
	return &a
}

func (b *Backups) keepAwake(a Awake) {
	raw, _ := json.MarshalIndent(a, "", "  ")
	if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil {
		b.Say("recording how long this machine was up since its last good night failed: %v", err)
	}
}

// newestGood is the newest good night of any module; zero before the first.
func newestGood(nights map[string]Night) time.Time {
	var newest time.Time
	for _, n := range nights {
		if n.OK && n.At.After(newest) {
			newest = n.At
		}
	}
	return newest
}

// Count adds one look: up is how long the machine was awake since the last one. A newer good night starts
// the count again. Pure.
func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake {
	if !good.Equal(a.From) {
		a.From, a.Up = good, 0
	}
	if up > 0 {
		a.Up += up
	}
	a.Looked = now.UTC()
	return a
}

// counting is the night's process's count, once a minute while it runs. It carries on from what an earlier
// process left, so a restart neither forgives an overdue night nor starts one fresh.
func counting(b *Backups) {
	a := Awake{}
	if was := b.LastAwake(); was != nil {
		a = *was
	}
	a.Started = time.Now().UTC()
	last := time.Now()
	for {
		now := time.Now()
		// Both readings carry the monotonic clock, which does not run while the machine is suspended.
		a = a.Count(newestGood(b.Nights()), now.Sub(last), now)
		a.Boot, a.Mono = machineClock()
		last = now
		b.keepAwake(a)
		time.Sleep(LookEvery)
	}
}