mesh-catalog/modules/restic/cmd/restic-backups/backups.go

// The machine's backups (novox/hq ADR 0214, to-be 43).
//
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
// directories already filled, into one file this module reads. Two kinds of line:
//
//	run <shell command>   run as root before the module's snapshot — a consistent dump of a store
//	path <directory>      a directory the module's snapshot keeps
//
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
//
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
// launching this binary runs as the operator's account, so restic and the run lines go through sudo
// without a prompt where the account is not root (ADR 0175 §4).
package main

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"sync"
	"time"
)

// Runner runs one command and answers what it printed, so the backups can be tested without restic
// or a store.
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// escalated is the command as it is run: as given when this process is root, else through sudo
// without a prompt.
func escalated(uid int, name string, args []string) (string, []string) {
	if uid == 0 {
		return name, args
	}
	return "sudo", append([]string{"-n", name}, args...)
}

// execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump
// that will not finish.
func execRunner(ctx context.Context, name string, args ...string) (string, error) {
	ctx, cancel := context.WithTimeout(ctx, 6*time.Hour)
	defer cancel()
	program, argv := escalated(os.Getuid(), name, args)
	var stdout, stderr bytes.Buffer
	cmd := exec.CommandContext(ctx, program, argv...)
	cmd.Stdout, cmd.Stderr = &stdout, &stderr
	if err := cmd.Run(); err != nil {
		said := strings.TrimSpace(stderr.String())
		if said == "" {
			said = strings.TrimSpace(stdout.String())
		}
		if program == "sudo" && strings.HasPrefix(said, "sudo:") {
			return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
		}
		lines := strings.Split(said, "\n")
		if len(lines) > 3 {
			lines = lines[len(lines)-3:]
		}
		if said == "" {
			return "", fmt.Errorf("%s: %w", name, err)
		}
		return "", errors.New(strings.Join(lines, " / "))
	}
	return stdout.String(), nil
}

// Declared is what one module declared.
type Declared struct {
	Module string   `json:"module"`
	Runs   []string `json:"runs"`
	Paths  []string `json:"paths"`
}

var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`)

// parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote
// them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a
// line this holder does not read is refused, naming the module, rather than skipped.
func parseDeclared(text string) ([]Declared, error) {
	var out []Declared
	current := -1
	for _, raw := range strings.Split(text, "\n") {
		line := strings.TrimSpace(raw)
		if line == "" {
			continue
		}
		if m := moduleHeader.FindStringSubmatch(line); m != nil {
			out = append(out, Declared{Module: m[1]})
			current = len(out) - 1
			continue
		}
		if strings.HasPrefix(line, "#") || current < 0 {
			continue
		}
		kind, value, _ := strings.Cut(line, " ")
		value = strings.TrimSpace(value)
		d := &out[current]
		switch {
		case kind == "run" && value != "":
			d.Runs = append(d.Runs, value)
		case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"):
			d.Paths = append(d.Paths, value)
		default:
			return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line)
		}
	}
	kept := out[:0]
	for _, d := range out {
		if len(d.Runs) > 0 || len(d.Paths) > 0 {
			kept = append(kept, d)
		}
	}
	return kept, nil
}

// Snapshot is one restore point, as restic lists it.
type Snapshot struct {
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
	Time     time.Time `json:"time"`
	Paths    []string  `json:"paths"`
	Tags     []string  `json:"tags"`
	Hostname string    `json:"hostname"`
}

// Night is how one module's last night went.
type Night struct {
	OK       bool      `json:"ok"`
	At       time.Time `json:"at"`
	Snapshot string    `json:"snapshot,omitempty"`
	Error    string    `json:"error,omitempty"`
}

// Keep is the rotation (novox/hq ADR 0214).
var Keep = struct{ Daily, Weekly, Monthly int }{14, 8, 6}

func tagOf(module string) string { return "module=" + module }

// Where is where the mesh put this module's things.
type Where struct {
	Declared, Repository, PasswordFile, State string
}

func whereFromEnv() (Where, error) {
	var missing []string
	get := func(k string) string {
		v := os.Getenv(k)
		if v == "" {
			missing = append(missing, k)
		}
		return v
	}
	w := Where{
		Declared:     get("MESH_BACKUP_DECLARED"),
		Repository:   get("MESH_BACKUP_REPOSITORY"),
		PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"),
		State:        get("MESH_BACKUP_STATE"),
	}
	if len(missing) > 0 {
		return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", "))
	}
	return w, nil
}

// Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never
// share a dump.
type Backups struct {
	Where Where
	Run   Runner
	Now   func() time.Time
	Say   func(format string, args ...any)
	mu    sync.Mutex
}

// exists is whether a path is there, asked as root: a store's directory is often its own user's
// alone (postgres's 0700 data directory), and the runtime's account asking for itself would see
// nothing — every such directory "missing", and its module never backed up.
func (b *Backups) exists(ctx context.Context, path string) bool {
	_, err := b.Run(ctx, "test", "-e", path)
	return err == nil
}

func (b *Backups) restic(ctx context.Context, args ...string) (string, error) {
	return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...)
}

// Declared is what the modules on this machine declared.
func (b *Backups) Declared() ([]Declared, error) {
	raw, err := os.ReadFile(b.Where.Declared)
	if err != nil {
		return nil, err
	}
	return parseDeclared(string(raw))
}

func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") }

// Nights is how each module's last night went.
func (b *Backups) Nights() map[string]Night {
	nights := map[string]Night{}
	if raw, err := os.ReadFile(b.nightsFile()); err == nil {
		_ = json.Unmarshal(raw, &nights)
	}
	return nights
}

func (b *Backups) record(module string, n Night) {
	nights := b.Nights()
	nights[module] = n
	raw, _ := json.MarshalIndent(nights, "", "  ")
	if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil {
		b.Say("recording %s's night failed: %v", module, err)
	}
}

var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`)

// ensureRepository makes the repository the first time. One that exists and cannot be opened is
// said, never replaced: replacing it would discard every restore point to fix a password.
func (b *Backups) ensureRepository(ctx context.Context) error {
	_, err := b.restic(ctx, "cat", "config")
	if err == nil {
		return nil
	}
	if !noRepository.MatchString(err.Error()) {
		return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err)
	}
	b.Say("no repository at %s; making one", b.Where.Repository)
	_, err = b.restic(ctx, "init")
	return err
}

// one is one module's night: its run lines, then one snapshot of its paths. A failure is that
// module's alone.
func (b *Backups) one(ctx context.Context, d Declared) Night {
	n := Night{At: b.Now().UTC()}
	fail := func(err error) Night {
		n.Error = err.Error()
		b.Say("%s: NOT backed up: %s", d.Module, n.Error)
		return n
	}
	for _, command := range d.Runs {
		if _, err := b.Run(ctx, "sh", "-c", command); err != nil {
			return fail(err)
		}
	}
	if len(d.Paths) == 0 {
		return fail(errors.New("it runs a dump and names no directory to keep it from"))
	}
	var missing []string
	for _, p := range d.Paths {
		if !b.exists(ctx, p) {
			missing = append(missing, p)
		}
	}
	if len(missing) > 0 {
		return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", ")))
	}
	out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...)
	if err != nil {
		return fail(err)
	}
	scanner := bufio.NewScanner(strings.NewReader(out))
	scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
	for scanner.Scan() {
		var m struct {
			MessageType string `json:"message_type"`
			SnapshotID  string `json:"snapshot_id"`
		}
		if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 {
			n.Snapshot = m.SnapshotID[:8]
		}
	}
	n.OK = true
	b.Say("%s: backed up (%s)", d.Module, n.Snapshot)
	return n
}

// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome.
func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if err := b.ensureRepository(ctx); err != nil {
		return nil, err
	}
	all, err := b.Declared()
	if err != nil {
		return nil, err
	}
	chosen := all
	if only != "" {
		chosen = nil
		var names []string
		for _, d := range all {
			names = append(names, d.Module)
			if d.Module == only {
				chosen = append(chosen, d)
			}
		}
		if len(chosen) == 0 {
			return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names))
		}
	}
	outcome := map[string]Night{}
	for _, d := range chosen {
		outcome[d.Module] = b.one(ctx, d)
		b.record(d.Module, outcome[d.Module])
	}
	if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
		"--keep-daily", fmt.Sprint(Keep.Daily), "--keep-weekly", fmt.Sprint(Keep.Weekly), "--keep-monthly", fmt.Sprint(Keep.Monthly)); err != nil {
		b.Say("thinning the restore points failed, and every one is kept: %v", err)
	}
	return outcome, nil
}

func orNothing(names []string) string {
	if len(names) == 0 {
		return "nothing"
	}
	return strings.Join(names, ", ")
}

// Snapshots is the restore points, of one module or all.
func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) {
	args := []string{"snapshots", "--json"}
	if module != "" {
		args = append(args, "--tag", tagOf(module))
	}
	out, err := b.restic(ctx, args...)
	if err != nil {
		return nil, err
	}
	var snaps []Snapshot
	if strings.TrimSpace(out) == "" {
		return nil, nil
	}
	if err := json.Unmarshal([]byte(out), &snaps); err != nil {
		return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err)
	}
	return snaps, nil
}

// ModuleBackups is what `backed-up` says about one module.
type ModuleBackups struct {
	Module        string    `json:"module"`
	Runs          int       `json:"runs"`
	Paths         []string  `json:"paths"`
	LastNight     *Night    `json:"lastNight"`
	RestorePoints int       `json:"restorePoints"`
	Newest        *Snapshot `json:"newest,omitempty"`
}

// BackedUp is what is backed up here: each module, what it declared, its last night and its restore
// points.
func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) {
	declared, err := b.Declared()
	if err != nil {
		return nil, err
	}
	nights := b.Nights()
	snaps, _ := b.Snapshots(ctx, module)
	out := []ModuleBackups{}
	for _, d := range declared {
		if module != "" && d.Module != module {
			continue
		}
		m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths}
		if n, ok := nights[d.Module]; ok {
			m.LastNight = &n
		}
		for _, s := range snaps {
			if slices.Contains(s.Tags, tagOf(d.Module)) {
				m.RestorePoints++
				newest := s
				m.Newest = &newest
			}
		}
		out = append(out, m)
	}
	return out, nil
}

// Restored is what a restore put where.
type Restored struct {
	Module   string   `json:"module"`
	From     Snapshot `json:"from"`
	Restored []string `json:"restored"`
	Live     string   `json:"live"`
}

// Restore puts a module's data from a restore point BESIDE the live data: each directory as
// <path>.restored-<stamp>. A target that already exists is refused, never overwritten.
func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	mine, err := b.Snapshots(ctx, module)
	if err != nil {
		return nil, err
	}
	if len(mine) == 0 {
		return nil, fmt.Errorf("%s has no restore point on this machine", module)
	}
	chosen := mine[len(mine)-1]
	if snapshot != "" {
		found := false
		var listed []string
		for _, s := range mine {
			listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339)))
			if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) {
				chosen, found = s, true
			}
		}
		if !found {
			return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", "))
		}
	}
	paths := chosen.Paths
	if path != "" {
		if !slices.Contains(chosen.Paths, path) {
			return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path)
		}
		paths = []string{path}
	}
	stamp := b.Now().UTC().Format("20060102-150405")
	r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"}
	for _, p := range paths {
		target := p + ".restored-" + stamp
		if b.exists(ctx, target) {
			return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target)
		}
		if _, err := b.restic(ctx, "restore", chosen.ID+":"+p, "--target", target); err != nil {
			return nil, err
		}
		r.Restored = append(r.Restored, target)
		b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target)
	}
	return r, nil
}

// Check is the weekly look at the repository's own integrity, with a sample of the data read back.
func (b *Backups) Check(ctx context.Context) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	_, err := b.restic(ctx, "check", "--read-data-subset", "5%")
	return err
}

// nextNight is when the next night is due: the given hour, local time, today while it is still
// ahead, else tomorrow.
func nextNight(now time.Time, hour int) time.Time {
	next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location())
	if !next.After(now) {
		next = next.AddDate(0, 0, 1)
	}
	return next
}

// missedANight is whether a night was missed: the newest good night of any module is older than a
// day and a bit — the machine was off, or this module was not running, at the hour.
func missedANight(nights map[string]Night, now time.Time) bool {
	var newest time.Time
	for _, n := range nights {
		if n.OK && n.At.After(newest) {
			newest = n.At
		}
	}
	return newest.IsZero() || now.Sub(newest) > 26*time.Hour
}