// The machine's backups (novox/hq ADR 0214, to-be 43). // // The mesh composes what to back up: every module on the machine contributes `backup` lines to the // node-backup seat, and the mesh writes them, each module's under a `# ` line and with its // directories already filled, into one file this module reads. Two kinds of line: // // run run as root before the module's snapshot — a consistent dump of a store // path a directory the module's snapshot keeps // // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and // restored on its own; one taken by hand is tagged as such too, and kept apart (hand.go). Everything lands in one repository on the machine — deduplicated, so every // night is a complete restore point and only what changed costs space — and is thinned by the machine's // settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters: // nothing leaves the machine. How the last night and thinning went is the module's health (health.go). // // Root's: the dumps read every store and the repository holds every module's data, and the runtime // launching this binary runs as the operator's account, so restic and the run lines go through sudo // without a prompt where the account is not root (ADR 0175 §4). package main import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "regexp" "slices" "strconv" "strings" "syscall" "time" ) // Runner runs one command and answers what it printed, so the backups can be tested without restic // or a store. type Runner func(ctx context.Context, name string, args ...string) (string, error) // escalated is the command as it is run: as given when this process is root, else through sudo // without a prompt. func escalated(uid int, name string, args []string) (string, []string) { if uid == 0 { return name, args } return "sudo", append([]string{"-n", name}, args...) } // execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump // that will not finish. func execRunner(ctx context.Context, name string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(ctx, 6*time.Hour) defer cancel() program, argv := escalated(os.Getuid(), name, args) var stdout, stderr bytes.Buffer cmd := exec.CommandContext(ctx, program, argv...) cmd.Stdout, cmd.Stderr = &stdout, &stderr if err := cmd.Run(); err != nil { said := strings.TrimSpace(stderr.String()) if said == "" { said = strings.TrimSpace(stdout.String()) } if program == "sudo" && strings.HasPrefix(said, "sudo:") { return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said) } lines := strings.Split(said, "\n") if len(lines) > 3 { lines = lines[len(lines)-3:] } if said == "" { return "", fmt.Errorf("%s: %w", name, err) } return "", errors.New(strings.Join(lines, " / ")) } return stdout.String(), nil } // Declared is what one module declared. type Declared struct { Module string `json:"module"` Runs []string `json:"runs"` Paths []string `json:"paths"` } var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`) // parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote // them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a // line this holder does not read is refused, naming the module, rather than skipped. func parseDeclared(text string) ([]Declared, error) { var out []Declared current := -1 for _, raw := range strings.Split(text, "\n") { line := strings.TrimSpace(raw) if line == "" { continue } if m := moduleHeader.FindStringSubmatch(line); m != nil { out = append(out, Declared{Module: m[1]}) current = len(out) - 1 continue } if strings.HasPrefix(line, "#") || current < 0 { continue } kind, value, _ := strings.Cut(line, " ") value = strings.TrimSpace(value) d := &out[current] switch { case kind == "run" && value != "": d.Runs = append(d.Runs, value) case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"): d.Paths = append(d.Paths, value) default: return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line) } } kept := out[:0] for _, d := range out { if len(d.Runs) > 0 || len(d.Paths) > 0 { kept = append(kept, d) } } return kept, nil } // Snapshot is one restore point, as restic lists it. type Snapshot struct { ID string `json:"id"` ShortID string `json:"short_id"` Time time.Time `json:"time"` Paths []string `json:"paths"` Tags []string `json:"tags"` Hostname string `json:"hostname"` } // Night is how one module's last night went. type Night struct { OK bool `json:"ok"` At time.Time `json:"at"` Snapshot string `json:"snapshot,omitempty"` Error string `json:"error,omitempty"` } func tagOf(module string) string { return "module=" + module } // Where is where the mesh put this module's things. type Where struct { Declared, Repository, PasswordFile, State string // Data is the composed file of every data item declared here (novox/hq ADR 0233); empty where the // module was placed by a definition older than it, and then nothing is measured. Data string } func whereFromEnv() (Where, error) { var missing []string get := func(k string) string { v := os.Getenv(k) if v == "" { missing = append(missing, k) } return v } w := Where{ Declared: get("MESH_BACKUP_DECLARED"), Repository: get("MESH_BACKUP_REPOSITORY"), PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"), State: get("MESH_BACKUP_STATE"), Data: os.Getenv("MESH_BACKUP_DATA"), } if len(missing) > 0 { return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", ")) } return w, nil } // Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never // share a dump — across the night's process and the tool runtime alike (hold). type Backups struct { Where Where Run Runner Now func() time.Time Say func(format string, args ...any) } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a // retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer // keeps the other out. const ( nightLock = "night.lock" measureLock = "measure.lock" ) // hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open // file's, so two holders in one process exclude each other as two processes do. func (b *Backups) hold(name string) (func(), error) { f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err) } for { err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX) if !errors.Is(err, syscall.EINTR) { break } } if err != nil { f.Close() return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err) } return func() { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) f.Close() }, nil } // exists is whether a path is there, asked as root: a store's directory is often its own user's // alone (postgres's 0700 data directory), and the runtime's account asking for itself would see // nothing — every such directory "missing", and its module never backed up. func (b *Backups) exists(ctx context.Context, path string) bool { _, err := b.Run(ctx, "test", "-e", path) return err == nil } func (b *Backups) restic(ctx context.Context, args ...string) (string, error) { return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...) } // Declared is what the modules on this machine declared. func (b *Backups) Declared() ([]Declared, error) { raw, err := os.ReadFile(b.Where.Declared) if err != nil { return nil, err } return parseDeclared(string(raw)) } func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") } // Nights is how each module's last night went. func (b *Backups) Nights() map[string]Night { nights := map[string]Night{} if raw, err := os.ReadFile(b.nightsFile()); err == nil { _ = json.Unmarshal(raw, &nights) } return nights } func (b *Backups) record(module string, n Night) { nights := b.Nights() nights[module] = n raw, _ := json.MarshalIndent(nights, "", " ") if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording %s's night failed: %v", module, err) } } var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`) // ensureRepository makes the repository the first time. One that exists and cannot be opened is // said, never replaced: replacing it would discard every restore point to fix a password. func (b *Backups) ensureRepository(ctx context.Context) error { _, err := b.restic(ctx, "cat", "config") if err == nil { return nil } if !noRepository.MatchString(err.Error()) { return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err) } b.Say("no repository at %s; making one", b.Where.Repository) _, err = b.restic(ctx, "init") return err } // one is one module's night: its run lines, then one snapshot of its paths. A failure is that // module's alone. func (b *Backups) one(ctx context.Context, d Declared, hand bool) Night { n := Night{At: b.Now().UTC()} fail := func(err error) Night { n.Error = err.Error() b.Say("%s: NOT backed up: %s", d.Module, n.Error) return n } for _, command := range d.Runs { if _, err := b.Run(ctx, "sh", "-c", command); err != nil { return fail(err) } } if len(d.Paths) == 0 { return fail(errors.New("it runs a dump and names no directory to keep it from")) } var missing []string for _, p := range d.Paths { if !b.exists(ctx, p) { missing = append(missing, p) } } if len(missing) > 0 { return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", "))) } args := []string{"backup", "--json", "--tag", tagOf(d.Module)} if hand { args = append(args, "--tag", HandTag) } out, err := b.restic(ctx, append(args, d.Paths...)...) if err != nil { return fail(err) } n.Snapshot = snapshotOf(out) n.OK = true b.recordGood(d.Module, n.At) b.Say("%s: backed up (%s)", d.Module, n.Snapshot) return n } // snapshotOf is the short id of the snapshot restic's JSON output says it made; empty when none. func snapshotOf(out string) string { id := "" scanner := bufio.NewScanner(strings.NewReader(out)) scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024) for scanner.Scan() { var m struct { MessageType string `json:"message_type"` SnapshotID string `json:"snapshot_id"` } if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 { id = m.SnapshotID[:8] } } return id } // BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and // keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { return b.backUp(ctx, only, false) } // backUp is a night, or a backup taken by hand (hand.go): the same but for the restore points' tag. func (b *Backups) backUp(ctx context.Context, only string, hand bool) (map[string]Night, error) { run := b.LastRun() run.At = b.Now().UTC() failed := func(err error) (map[string]Night, error) { run.Failed = err.Error() b.keepRun(run) return nil, err } release, err := b.hold(nightLock) if err != nil { return failed(err) } defer release() if err := b.ensureRepository(ctx); err != nil { return failed(err) } all, err := b.Declared() if err != nil { return failed(err) } chosen := all if only != "" { chosen = nil var names []string for _, d := range all { names = append(names, d.Module) if d.Module == only { chosen = append(chosen, d) } } if len(chosen) == 0 { // Asked of one module that is not here: nothing ran, and the last night stands as it was. return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names)) } } outcome := map[string]Night{} for _, d := range chosen { outcome[d.Module] = b.one(ctx, d, hand) b.record(d.Module, outcome[d.Module]) } // A retention that does not read thins nothing: keeping too much costs space, thinning by a value // nobody meant loses restore points (novox/hq issue 416). thinned := Thinning{At: b.Now().UTC()} keep, err := b.retention(ctx) if err != nil { thinned.Refused = err.Error() b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err) } else if letGo, err := b.thin(ctx, keep); err != nil { thinned.Kept, thinned.Error = &keep, err.Error() b.Say("thinning the restore points failed, and every one is kept: %v", err) } else { thinned.Kept, thinned.OK, thinned.LetGo = &keep, true, letGo } run.Failed, run.Thinned = "", &thinned b.keepRun(run) return outcome, nil } // thin forgets the restore points taken by hand that are past their days and have a later night of their // module, by id, then thins the rest by the counts, keeping every one still tagged as taken by hand (novox/hq // issue 453). It answers the short ids it let go. A list that does not read thins nothing: a restore point // taken by hand is never lost to a list nobody could read. func (b *Backups) thin(ctx context.Context, keep Retention) ([]string, error) { snaps, err := b.Snapshots(ctx, "") if err != nil { return nil, fmt.Errorf("the restore points could not be listed: %w", err) } var ids, letGo []string for _, s := range handLetGo(snaps, b.Now(), keep.Hand) { ids, letGo = append(ids, s.ID), append(letGo, s.ShortID) } if len(ids) > 0 { if _, err := b.restic(ctx, append([]string{"forget"}, ids...)...); err != nil { return nil, err } b.Say("let go the restore points taken by hand past %d days with a later night: %s", keep.Hand, strings.Join(letGo, ", ")) } _, err = b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", "--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly), "--keep-tag", HandTag) return letGo, err } // Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its // thinning. Each module's own outcome is its Night. type Run struct { At time.Time `json:"at"` // Failed is why it did not run at all: the repository would not open, or what to back up does not read. Failed string `json:"failed,omitempty"` Thinned *Thinning `json:"thinned,omitempty"` } // Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing. type Thinning struct { At time.Time `json:"at"` OK bool `json:"ok"` Kept *Retention `json:"kept,omitempty"` Refused string `json:"refused,omitempty"` Error string `json:"error,omitempty"` // LetGo is the restore points taken by hand it forgot, past their days and with a later night (hand.go). LetGo []string `json:"letGo,omitempty"` } func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") } // LastRun is how the last night went as a whole; empty before the first. func (b *Backups) LastRun() Run { var r Run if raw, err := os.ReadFile(b.runFile()); err == nil { _ = json.Unmarshal(raw, &r) } return r } func (b *Backups) keepRun(r Run) { raw, _ := json.MarshalIndent(r, "", " ") if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording how the night went failed: %v", err) } } func orNothing(names []string) string { if len(names) == 0 { return "nothing" } return strings.Join(names, ", ") } // Snapshots is the restore points, of one module or all. func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) { args := []string{"snapshots", "--json"} if module != "" { args = append(args, "--tag", tagOf(module)) } out, err := b.restic(ctx, args...) if err != nil { return nil, err } var snaps []Snapshot if strings.TrimSpace(out) == "" { return nil, nil } if err := json.Unmarshal([]byte(out), &snaps); err != nil { return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err) } return snaps, nil } // ModuleBackups is what `backed-up` says about one module. type ModuleBackups struct { Module string `json:"module"` Runs int `json:"runs"` Paths []string `json:"paths"` LastNight *Night `json:"lastNight"` RestorePoints int `json:"restorePoints"` Newest *Snapshot `json:"newest,omitempty"` // Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned // how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its // own, so the answer stays the list the controller reads. Retention InForce `json:"retention"` Thinned *Thinning `json:"thinned,omitempty"` // Data is every item the module declares, measured, with its newest good backup (novox/hq ADR // 0233): what the controller's self-check reads. Data []ItemReport `json:"data"` // HandTaken is its restore points taken by hand, each with until when it is kept (novox/hq issue 453). HandTaken []HandKept `json:"handTaken"` } // BackedUp is what is backed up here: each module, what it declared, its last night and its restore // points. func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) { declared, err := b.Declared() if err != nil { return nil, err } items, err := b.Items() if err != nil { return nil, err } nights := b.Nights() good := b.Good() measured := b.Measurements() snaps, _ := b.Snapshots(ctx, module) inForce := b.inForce(ctx) thinned := b.LastRun().Thinned out := []ModuleBackups{} listed := map[string]bool{} for _, d := range declared { if module != "" && d.Module != module { continue } listed[d.Module] = true m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned, Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module]), HandTaken: handKept(d.Module, snaps, inForce)} if n, ok := nights[d.Module]; ok { m.LastNight = &n } for _, s := range snaps { if slices.Contains(s.Tags, tagOf(d.Module)) { m.RestorePoints++ newest := s m.Newest = &newest } } out = append(out, m) } // A module declaring data and nothing to back up is still measured, and said. for name, its := range items { if listed[name] || (module != "" && name != module) { continue } out = append(out, ModuleBackups{Module: name, Paths: []string{}, HandTaken: []HandKept{}, Data: itemReports(its, measured[name], nil, time.Time{})}) } return out, nil } // Restored is what a restore put where. type Restored struct { Module string `json:"module"` From Snapshot `json:"from"` Restored []string `json:"restored"` Live string `json:"live"` } // Restore puts a module's data from a restore point BESIDE the live data: each directory as // .restored- — or, where a directory on the way belongs to another account, in this module's // state as restored/// (besideIsUnsafe). A target that already exists is refused, never // overwritten. func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) { release, err := b.hold(nightLock) if err != nil { return nil, err } defer release() mine, err := b.Snapshots(ctx, module) if err != nil { return nil, err } if len(mine) == 0 { return nil, fmt.Errorf("%s has no restore point on this machine", module) } chosen := mine[len(mine)-1] if snapshot != "" { found := false var listed []string for _, s := range mine { listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339))) if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) { chosen, found = s, true } } if !found { return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", ")) } } paths := chosen.Paths if path != "" { if !slices.Contains(chosen.Paths, path) { return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path) } paths = []string{path} } stamp := b.Now().UTC().Format("20060102-150405") r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"} for _, p := range paths { target := p + ".restored-" + stamp if why := b.besideIsUnsafe(ctx, target); why != "" { // Root writes the restore. Beside a path another account controls, that account could put a // symbolic link where root is about to write and have root write wherever it points (novox/hq ADR // 0266: the agent's own home is such a place). Kept in this module's state instead, which only // root and the runtime's account reach. target = filepath.Join(b.Where.State, "restored", stamp, module) + p if _, err := b.Run(ctx, "mkdir", "-p", "-m", "0700", filepath.Dir(target)); err != nil { return nil, fmt.Errorf("%s: %s, and the place in this module's state could not be made: %w", p, why, err) } b.Say("%s: %s; restoring into this module's state instead", p, why) } if b.exists(ctx, target) { return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target) } if err := b.restoreOne(ctx, chosen.ID, p, target); err != nil { return nil, err } r.Restored = append(r.Restored, target) b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target) } return r, nil } // besideIsUnsafe says why root must not restore at target, beside the live data: a directory on the way to it // is a symbolic link, belongs to an account other than root and the one this process runs as, or may be written // by its group or everyone — any of which could swap a link in while root writes. Empty when every directory on the way is root's or the runtime's. func (b *Backups) besideIsUnsafe(ctx context.Context, target string) string { me := strconv.Itoa(os.Getuid()) dir := filepath.Dir(filepath.Clean(target)) var chain []string for d := dir; ; d = filepath.Dir(d) { chain = append(chain, d) if d == "/" || d == "." { break } } for i := len(chain) - 1; i >= 0; i-- { out, err := b.Run(ctx, "stat", "-c", "%u %a %F", chain[i]) if err != nil { return chain[i] + " could not be read" } owner, rest, _ := strings.Cut(strings.TrimSpace(out), " ") perm, kind, _ := strings.Cut(rest, " ") if kind == "symbolic link" { return chain[i] + " is a symbolic link" } if owner != "0" && owner != me { return chain[i] + " belongs to another account" } // Another account may write into a directory open to its group or to everyone, sticky or not, and so // put a link where root is about to write. mode, err := strconv.ParseUint(perm, 8, 32) if err != nil { return chain[i] + "'s mode could not be read" } if mode&0o022 != 0 { return chain[i] + " may be written by other accounts" } } return "" } // restoreOne puts one kept path at target. A directory is restored as the snapshot's subfolder, so // its contents land directly in target; a single file cannot be — restic restores a subfolder, not a // file (a settings file refused with "not a directory" on the first restore of an app, 2026-10-05) — // so it is restored with its full path into a scratch directory beside target, moved into place, and // the scratch directory removed. func (b *Backups) restoreOne(ctx context.Context, id, path, target string) error { file, err := b.isFile(ctx, id, path) if err != nil { return err } if !file { _, err := b.restic(ctx, "restore", id+":"+path, "--target", target) return err } scratch := target + ".partial" if _, err := b.restic(ctx, "restore", id, "--target", scratch, "--include", path); err != nil { return err } if _, err := b.Run(ctx, "mv", scratch+path, target); err != nil { return err } _, err = b.Run(ctx, "rm", "-rf", scratch) return err } // isFile is whether a kept path is a single file in the snapshot, as restic lists it. func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) { out, err := b.restic(ctx, "ls", "--json", id, path) if err != nil { return false, err } scanner := bufio.NewScanner(strings.NewReader(out)) scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024) for scanner.Scan() { var node struct { Type string `json:"type"` Path string `json:"path"` } if json.Unmarshal(scanner.Bytes(), &node) == nil && node.Path == path { return node.Type == "file", nil } } return false, fmt.Errorf("restore point %s does not list %s", id, path) } // Check is the weekly look at the repository's own integrity, with a sample of the data read back. func (b *Backups) Check(ctx context.Context) error { release, err := b.hold(nightLock) if err != nil { return err } defer release() _, err = b.restic(ctx, "check", "--read-data-subset", "5%") return err } // nextNight is when the next night is due: the given hour, local time, today while it is still // ahead, else tomorrow. func nextNight(now time.Time, hour int) time.Time { next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location()) if !next.After(now) { next = next.AddDate(0, 0, 1) } return next } // missedANight is whether a night was missed: the newest good night of any module is older than a // day and a bit — the machine was off, or this module was not running, at the hour. func missedANight(nights map[string]Night, now time.Time) bool { newest := newestGood(nights) return newest.IsZero() || now.Sub(newest) > Overdue }