mesh-catalog/modules/restic/cmd/restic-backups/backups.go

// The machine's backups (novox/hq ADR 0214, to-be 43).
//
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
// directories already filled, into one file this module reads. Two kinds of line:
//
//	run <shell command>   run as root before the module's snapshot — a consistent dump of a store
//	path <directory>      a directory the module's snapshot keeps
//
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
// night is a complete restore point and only what changed costs space — and is thinned by the machine's
// settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters:
// nothing leaves the machine. How the last night and thinning went is the module's health (health.go).
//
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
// launching this binary runs as the operator's account, so restic and the run lines go through sudo
// without a prompt where the account is not root (ADR 0175 §4).
package main

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"syscall"
	"time"
)

// Runner runs one command and answers what it printed, so the backups can be tested without restic
// or a store.
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// escalated is the command as it is run: as given when this process is root, else through sudo
// without a prompt.
func escalated(uid int, name string, args []string) (string, []string) {
	if uid == 0 {
		return name, args
	}
	return "sudo", append([]string{"-n", name}, args...)
}

// execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump
// that will not finish.
func execRunner(ctx context.Context, name string, args ...string) (string, error) {
	ctx, cancel := context.WithTimeout(ctx, 6*time.Hour)
	defer cancel()
	program, argv := escalated(os.Getuid(), name, args)
	var stdout, stderr bytes.Buffer
	cmd := exec.CommandContext(ctx, program, argv...)
	cmd.Stdout, cmd.Stderr = &stdout, &stderr
	if err := cmd.Run(); err != nil {
		said := strings.TrimSpace(stderr.String())
		if said == "" {
			said = strings.TrimSpace(stdout.String())
		}
		if program == "sudo" && strings.HasPrefix(said, "sudo:") {
			return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
		}
		lines := strings.Split(said, "\n")
		if len(lines) > 3 {
			lines = lines[len(lines)-3:]
		}
		if said == "" {
			return "", fmt.Errorf("%s: %w", name, err)
		}
		return "", errors.New(strings.Join(lines, " / "))
	}
	return stdout.String(), nil
}

// Declared is what one module declared.
type Declared struct {
	Module string   `json:"module"`
	Runs   []string `json:"runs"`
	Paths  []string `json:"paths"`
}

var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`)

// parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote
// them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a
// line this holder does not read is refused, naming the module, rather than skipped.
func parseDeclared(text string) ([]Declared, error) {
	var out []Declared
	current := -1
	for _, raw := range strings.Split(text, "\n") {
		line := strings.TrimSpace(raw)
		if line == "" {
			continue
		}
		if m := moduleHeader.FindStringSubmatch(line); m != nil {
			out = append(out, Declared{Module: m[1]})
			current = len(out) - 1
			continue
		}
		if strings.HasPrefix(line, "#") || current < 0 {
			continue
		}
		kind, value, _ := strings.Cut(line, " ")
		value = strings.TrimSpace(value)
		d := &out[current]
		switch {
		case kind == "run" && value != "":
			d.Runs = append(d.Runs, value)
		case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"):
			d.Paths = append(d.Paths, value)
		default:
			return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line)
		}
	}
	kept := out[:0]
	for _, d := range out {
		if len(d.Runs) > 0 || len(d.Paths) > 0 {
			kept = append(kept, d)
		}
	}
	return kept, nil
}

// Snapshot is one restore point, as restic lists it.
type Snapshot struct {
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
	Time     time.Time `json:"time"`
	Paths    []string  `json:"paths"`
	Tags     []string  `json:"tags"`
	Hostname string    `json:"hostname"`
}

// Night is how one module's last night went.
type Night struct {
	OK       bool      `json:"ok"`
	At       time.Time `json:"at"`
	Snapshot string    `json:"snapshot,omitempty"`
	Error    string    `json:"error,omitempty"`
}

func tagOf(module string) string { return "module=" + module }

// Where is where the mesh put this module's things.
type Where struct {
	Declared, Repository, PasswordFile, State string
	// Data is the composed file of every data item declared here (novox/hq ADR 0233); empty where the
	// module was placed by a definition older than it, and then nothing is measured.
	Data string
}

func whereFromEnv() (Where, error) {
	var missing []string
	get := func(k string) string {
		v := os.Getenv(k)
		if v == "" {
			missing = append(missing, k)
		}
		return v
	}
	w := Where{
		Declared:     get("MESH_BACKUP_DECLARED"),
		Repository:   get("MESH_BACKUP_REPOSITORY"),
		PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"),
		State:        get("MESH_BACKUP_STATE"),
		Data:         os.Getenv("MESH_BACKUP_DATA"),
	}
	if len(missing) > 0 {
		return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", "))
	}
	return w, nil
}

// Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never
// share a dump — across the night's process and the tool runtime alike (hold).
type Backups struct {
	Where Where
	Run   Runner
	Now   func() time.Time
	Say   func(format string, args ...any)
}

// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a
// retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer
// keeps the other out.
const (
	nightLock   = "night.lock"
	measureLock = "measure.lock"
)

// hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open
// file's, so two holders in one process exclude each other as two processes do.
func (b *Backups) hold(name string) (func(), error) {
	f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600)
	if err != nil {
		return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err)
	}
	for {
		err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX)
		if !errors.Is(err, syscall.EINTR) {
			break
		}
	}
	if err != nil {
		f.Close()
		return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err)
	}
	return func() {
		_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
		f.Close()
	}, nil
}

// exists is whether a path is there, asked as root: a store's directory is often its own user's
// alone (postgres's 0700 data directory), and the runtime's account asking for itself would see
// nothing — every such directory "missing", and its module never backed up.
func (b *Backups) exists(ctx context.Context, path string) bool {
	_, err := b.Run(ctx, "test", "-e", path)
	return err == nil
}

func (b *Backups) restic(ctx context.Context, args ...string) (string, error) {
	return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...)
}

// Declared is what the modules on this machine declared.
func (b *Backups) Declared() ([]Declared, error) {
	raw, err := os.ReadFile(b.Where.Declared)
	if err != nil {
		return nil, err
	}
	return parseDeclared(string(raw))
}

func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") }

// Nights is how each module's last night went.
func (b *Backups) Nights() map[string]Night {
	nights := map[string]Night{}
	if raw, err := os.ReadFile(b.nightsFile()); err == nil {
		_ = json.Unmarshal(raw, &nights)
	}
	return nights
}

func (b *Backups) record(module string, n Night) {
	nights := b.Nights()
	nights[module] = n
	raw, _ := json.MarshalIndent(nights, "", "  ")
	if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil {
		b.Say("recording %s's night failed: %v", module, err)
	}
}

var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`)

// ensureRepository makes the repository the first time. One that exists and cannot be opened is
// said, never replaced: replacing it would discard every restore point to fix a password.
func (b *Backups) ensureRepository(ctx context.Context) error {
	_, err := b.restic(ctx, "cat", "config")
	if err == nil {
		return nil
	}
	if !noRepository.MatchString(err.Error()) {
		return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err)
	}
	b.Say("no repository at %s; making one", b.Where.Repository)
	_, err = b.restic(ctx, "init")
	return err
}

// one is one module's night: its run lines, then one snapshot of its paths. A failure is that
// module's alone.
func (b *Backups) one(ctx context.Context, d Declared) Night {
	n := Night{At: b.Now().UTC()}
	fail := func(err error) Night {
		n.Error = err.Error()
		b.Say("%s: NOT backed up: %s", d.Module, n.Error)
		return n
	}
	for _, command := range d.Runs {
		if _, err := b.Run(ctx, "sh", "-c", command); err != nil {
			return fail(err)
		}
	}
	if len(d.Paths) == 0 {
		return fail(errors.New("it runs a dump and names no directory to keep it from"))
	}
	var missing []string
	for _, p := range d.Paths {
		if !b.exists(ctx, p) {
			missing = append(missing, p)
		}
	}
	if len(missing) > 0 {
		return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", ")))
	}
	out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...)
	if err != nil {
		return fail(err)
	}
	n.Snapshot = snapshotOf(out)
	n.OK = true
	b.recordGood(d.Module, n.At)
	b.Say("%s: backed up (%s)", d.Module, n.Snapshot)
	return n
}

// snapshotOf is the short id of the snapshot restic's JSON output says it made; empty when none.
func snapshotOf(out string) string {
	id := ""
	scanner := bufio.NewScanner(strings.NewReader(out))
	scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
	for scanner.Scan() {
		var m struct {
			MessageType string `json:"message_type"`
			SnapshotID  string `json:"snapshot_id"`
		}
		if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 {
			id = m.SnapshotID[:8]
		}
	}
	return id
}

// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and
// keeps how the night as a whole and its thinning went for the health (novox/hq issue 454).
func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) {
	run := b.LastRun()
	run.At = b.Now().UTC()
	failed := func(err error) (map[string]Night, error) {
		run.Failed = err.Error()
		b.keepRun(run)
		return nil, err
	}
	release, err := b.hold(nightLock)
	if err != nil {
		return failed(err)
	}
	defer release()
	if err := b.ensureRepository(ctx); err != nil {
		return failed(err)
	}
	all, err := b.Declared()
	if err != nil {
		return failed(err)
	}
	chosen := all
	if only != "" {
		chosen = nil
		var names []string
		for _, d := range all {
			names = append(names, d.Module)
			if d.Module == only {
				chosen = append(chosen, d)
			}
		}
		if len(chosen) == 0 {
			// Asked of one module that is not here: nothing ran, and the last night stands as it was.
			return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names))
		}
	}
	outcome := map[string]Night{}
	for _, d := range chosen {
		outcome[d.Module] = b.one(ctx, d)
		b.record(d.Module, outcome[d.Module])
	}
	// A retention that does not read thins nothing: keeping too much costs space, thinning by a value
	// nobody meant loses restore points (novox/hq issue 416).
	thinned := Thinning{At: b.Now().UTC()}
	keep, err := b.retention(ctx)
	if err != nil {
		thinned.Refused = err.Error()
		b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err)
	} else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
		"--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil {
		thinned.Kept, thinned.Error = &keep, err.Error()
		b.Say("thinning the restore points failed, and every one is kept: %v", err)
	} else {
		thinned.Kept, thinned.OK = &keep, true
	}
	run.Failed, run.Thinned = "", &thinned
	b.keepRun(run)
	return outcome, nil
}

// Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its
// thinning. Each module's own outcome is its Night.
type Run struct {
	At time.Time `json:"at"`
	// Failed is why it did not run at all: the repository would not open, or what to back up does not read.
	Failed  string    `json:"failed,omitempty"`
	Thinned *Thinning `json:"thinned,omitempty"`
}

// Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing.
type Thinning struct {
	At      time.Time  `json:"at"`
	OK      bool       `json:"ok"`
	Kept    *Retention `json:"kept,omitempty"`
	Refused string     `json:"refused,omitempty"`
	Error   string     `json:"error,omitempty"`
}

func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") }

// LastRun is how the last night went as a whole; empty before the first.
func (b *Backups) LastRun() Run {
	var r Run
	if raw, err := os.ReadFile(b.runFile()); err == nil {
		_ = json.Unmarshal(raw, &r)
	}
	return r
}

func (b *Backups) keepRun(r Run) {
	raw, _ := json.MarshalIndent(r, "", "  ")
	if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil {
		b.Say("recording how the night went failed: %v", err)
	}
}

func orNothing(names []string) string {
	if len(names) == 0 {
		return "nothing"
	}
	return strings.Join(names, ", ")
}

// Snapshots is the restore points, of one module or all.
func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) {
	args := []string{"snapshots", "--json"}
	if module != "" {
		args = append(args, "--tag", tagOf(module))
	}
	out, err := b.restic(ctx, args...)
	if err != nil {
		return nil, err
	}
	var snaps []Snapshot
	if strings.TrimSpace(out) == "" {
		return nil, nil
	}
	if err := json.Unmarshal([]byte(out), &snaps); err != nil {
		return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err)
	}
	return snaps, nil
}

// ModuleBackups is what `backed-up` says about one module.
type ModuleBackups struct {
	Module        string    `json:"module"`
	Runs          int       `json:"runs"`
	Paths         []string  `json:"paths"`
	LastNight     *Night    `json:"lastNight"`
	RestorePoints int       `json:"restorePoints"`
	Newest        *Snapshot `json:"newest,omitempty"`
	// Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned
	// how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its
	// own, so the answer stays the list the controller reads.
	Retention InForce   `json:"retention"`
	Thinned   *Thinning `json:"thinned,omitempty"`
	// Data is every item the module declares, measured, with its newest good backup (novox/hq ADR
	// 0233): what the controller's self-check reads.
	Data []ItemReport `json:"data"`
}

// BackedUp is what is backed up here: each module, what it declared, its last night and its restore
// points.
func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) {
	declared, err := b.Declared()
	if err != nil {
		return nil, err
	}
	items, err := b.Items()
	if err != nil {
		return nil, err
	}
	nights := b.Nights()
	good := b.Good()
	measured := b.Measurements()
	snaps, _ := b.Snapshots(ctx, module)
	inForce := b.inForce(ctx)
	thinned := b.LastRun().Thinned
	out := []ModuleBackups{}
	listed := map[string]bool{}
	for _, d := range declared {
		if module != "" && d.Module != module {
			continue
		}
		listed[d.Module] = true
		m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned,
			Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])}
		if n, ok := nights[d.Module]; ok {
			m.LastNight = &n
		}
		for _, s := range snaps {
			if slices.Contains(s.Tags, tagOf(d.Module)) {
				m.RestorePoints++
				newest := s
				m.Newest = &newest
			}
		}
		out = append(out, m)
	}
	// A module declaring data and nothing to back up is still measured, and said.
	for name, its := range items {
		if listed[name] || (module != "" && name != module) {
			continue
		}
		out = append(out, ModuleBackups{Module: name, Paths: []string{}, Data: itemReports(its, measured[name], nil, time.Time{})})
	}
	return out, nil
}

// Restored is what a restore put where.
type Restored struct {
	Module   string   `json:"module"`
	From     Snapshot `json:"from"`
	Restored []string `json:"restored"`
	Live     string   `json:"live"`
}

// Restore puts a module's data from a restore point BESIDE the live data: each directory as
// <path>.restored-<stamp> — or, where a directory on the way belongs to another account, in this module's
// state as restored/<stamp>/<module>/<path> (besideIsUnsafe). A target that already exists is refused, never
// overwritten.
func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) {
	release, err := b.hold(nightLock)
	if err != nil {
		return nil, err
	}
	defer release()
	mine, err := b.Snapshots(ctx, module)
	if err != nil {
		return nil, err
	}
	if len(mine) == 0 {
		return nil, fmt.Errorf("%s has no restore point on this machine", module)
	}
	chosen := mine[len(mine)-1]
	if snapshot != "" {
		found := false
		var listed []string
		for _, s := range mine {
			listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339)))
			if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) {
				chosen, found = s, true
			}
		}
		if !found {
			return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", "))
		}
	}
	paths := chosen.Paths
	if path != "" {
		if !slices.Contains(chosen.Paths, path) {
			return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path)
		}
		paths = []string{path}
	}
	stamp := b.Now().UTC().Format("20060102-150405")
	r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"}
	for _, p := range paths {
		target := p + ".restored-" + stamp
		if why := b.besideIsUnsafe(ctx, target); why != "" {
			// Root writes the restore. Beside a path another account controls, that account could put a
			// symbolic link where root is about to write and have root write wherever it points (novox/hq ADR
			// 0266: the agent's own home is such a place). Kept in this module's state instead, which only
			// root and the runtime's account reach.
			target = filepath.Join(b.Where.State, "restored", stamp, module) + p
			if _, err := b.Run(ctx, "mkdir", "-p", "-m", "0700", filepath.Dir(target)); err != nil {
				return nil, fmt.Errorf("%s: %s, and the place in this module's state could not be made: %w", p, why, err)
			}
			b.Say("%s: %s; restoring into this module's state instead", p, why)
		}
		if b.exists(ctx, target) {
			return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target)
		}
		if err := b.restoreOne(ctx, chosen.ID, p, target); err != nil {
			return nil, err
		}
		r.Restored = append(r.Restored, target)
		b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target)
	}
	return r, nil
}

// besideIsUnsafe says why root must not restore at target, beside the live data: a directory on the way to it
// is a symbolic link, belongs to an account other than root and the one this process runs as, or may be written
// by its group or everyone — any of which could swap a link in while root writes. Empty when every directory on the way is root's or the runtime's.
func (b *Backups) besideIsUnsafe(ctx context.Context, target string) string {
	me := strconv.Itoa(os.Getuid())
	dir := filepath.Dir(filepath.Clean(target))
	var chain []string
	for d := dir; ; d = filepath.Dir(d) {
		chain = append(chain, d)
		if d == "/" || d == "." {
			break
		}
	}
	for i := len(chain) - 1; i >= 0; i-- {
		out, err := b.Run(ctx, "stat", "-c", "%u %a %F", chain[i])
		if err != nil {
			return chain[i] + " could not be read"
		}
		owner, rest, _ := strings.Cut(strings.TrimSpace(out), " ")
		perm, kind, _ := strings.Cut(rest, " ")
		if kind == "symbolic link" {
			return chain[i] + " is a symbolic link"
		}
		if owner != "0" && owner != me {
			return chain[i] + " belongs to another account"
		}
		// Another account may write into a directory open to its group or to everyone, sticky or not, and so
		// put a link where root is about to write.
		mode, err := strconv.ParseUint(perm, 8, 32)
		if err != nil {
			return chain[i] + "'s mode could not be read"
		}
		if mode&0o022 != 0 {
			return chain[i] + " may be written by other accounts"
		}
	}
	return ""
}

// restoreOne puts one kept path at target. A directory is restored as the snapshot's subfolder, so
// its contents land directly in target; a single file cannot be — restic restores a subfolder, not a
// file (a settings file refused with "not a directory" on the first restore of an app, 2026-10-05) —
// so it is restored with its full path into a scratch directory beside target, moved into place, and
// the scratch directory removed.
func (b *Backups) restoreOne(ctx context.Context, id, path, target string) error {
	file, err := b.isFile(ctx, id, path)
	if err != nil {
		return err
	}
	if !file {
		_, err := b.restic(ctx, "restore", id+":"+path, "--target", target)
		return err
	}
	scratch := target + ".partial"
	if _, err := b.restic(ctx, "restore", id, "--target", scratch, "--include", path); err != nil {
		return err
	}
	if _, err := b.Run(ctx, "mv", scratch+path, target); err != nil {
		return err
	}
	_, err = b.Run(ctx, "rm", "-rf", scratch)
	return err
}

// isFile is whether a kept path is a single file in the snapshot, as restic lists it.
func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) {
	out, err := b.restic(ctx, "ls", "--json", id, path)
	if err != nil {
		return false, err
	}
	scanner := bufio.NewScanner(strings.NewReader(out))
	scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
	for scanner.Scan() {
		var node struct {
			Type string `json:"type"`
			Path string `json:"path"`
		}
		if json.Unmarshal(scanner.Bytes(), &node) == nil && node.Path == path {
			return node.Type == "file", nil
		}
	}
	return false, fmt.Errorf("restore point %s does not list %s", id, path)
}

// Check is the weekly look at the repository's own integrity, with a sample of the data read back.
func (b *Backups) Check(ctx context.Context) error {
	release, err := b.hold(nightLock)
	if err != nil {
		return err
	}
	defer release()
	_, err = b.restic(ctx, "check", "--read-data-subset", "5%")
	return err
}

// nextNight is when the next night is due: the given hour, local time, today while it is still
// ahead, else tomorrow.
func nextNight(now time.Time, hour int) time.Time {
	next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location())
	if !next.After(now) {
		next = next.AddDate(0, 0, 1)
	}
	return next
}

// missedANight is whether a night was missed: the newest good night of any module is older than a
// day and a bit — the machine was off, or this module was not running, at the hour.
func missedANight(nights map[string]Night, now time.Time) bool {
	newest := newestGood(nights)
	return newest.IsZero() || now.Sub(newest) > Overdue
}