diff --git a/modules/restic/cmd/restic-backups/awake.go b/modules/restic/cmd/restic-backups/awake.go new file mode 100644 index 00000000..d11ede4c --- /dev/null +++ b/modules/restic/cmd/restic-backups/awake.go @@ -0,0 +1,102 @@ +package main + +// A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a +// restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed +// night behind, so the health also asks how long this machine has been up, with the night's process running, +// since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has +// missed nothing it could have done, and the night's process runs one soon after it starts again. +// +// The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while +// the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time +// the process was not running is not counted either: that is its unit's to say, judged by liveness. + +import ( + "encoding/json" + "os" + "path/filepath" + "time" +) + +// Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours +// a night may take or be late by. +const Overdue = 26 * time.Hour + +// LookEvery is how often the night's process counts; Stale how old its count may be before it is said. +const ( + LookEvery = time.Minute + Stale = 10 * time.Minute +) + +// Awake is the night's process's count: since which good night (zero before the first), how long the +// machine was up with the process running, when the process started and when it last counted. +type Awake struct { + From time.Time `json:"from"` + Up time.Duration `json:"up"` + Started time.Time `json:"started"` + Looked time.Time `json:"looked"` +} + +func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") } + +// LastAwake is the count as the night's process last wrote it; nil before it ever did. +func (b *Backups) LastAwake() *Awake { + raw, err := os.ReadFile(b.awakeFile()) + if err != nil { + return nil + } + var a Awake + if json.Unmarshal(raw, &a) != nil { + return nil + } + return &a +} + +func (b *Backups) keepAwake(a Awake) { + raw, _ := json.MarshalIndent(a, "", " ") + if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording how long this machine was up since its last good night failed: %v", err) + } +} + +// newestGood is the newest good night of any module; zero before the first. +func newestGood(nights map[string]Night) time.Time { + var newest time.Time + for _, n := range nights { + if n.OK && n.At.After(newest) { + newest = n.At + } + } + return newest +} + +// Count adds one look: up is how long the machine was awake since the last one. A newer good night starts +// the count again. Pure. +func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake { + if !good.Equal(a.From) { + a.From, a.Up = good, 0 + } + if up > 0 { + a.Up += up + } + a.Looked = now.UTC() + return a +} + +// counting is the night's process's count, once a minute while it runs. It carries on from what an earlier +// process left, so a restart neither forgives an overdue night nor starts one fresh. +func counting(b *Backups) { + a := Awake{} + if was := b.LastAwake(); was != nil { + a = *was + } + a.Started = time.Now().UTC() + last := time.Now() + for { + now := time.Now() + // Both readings carry the monotonic clock, which does not run while the machine is suspended. + a = a.Count(newestGood(b.Nights()), now.Sub(last), now) + last = now + b.keepAwake(a) + time.Sleep(LookEvery) + } +} diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 3b67e781..aba6086e 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -9,9 +9,9 @@ // // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and // restored on its own. Everything lands in one repository on the machine — deduplicated, so every -// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, -// 8 weekly and 6 monthly unless the machine's settings say otherwise (retention.go). Against -// mistakes, not disasters: nothing leaves the machine. +// night is a complete restore point and only what changed costs space — and is thinned by the machine's +// settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters: +// nothing leaves the machine. How the last night and thinning went is the module's health (health.go). // // Root's: the dumps read every store and the repository holds every module's data, and the runtime // launching this binary runs as the operator's account, so restic and the run lines go through sudo @@ -32,7 +32,7 @@ import ( "slices" "strconv" "strings" - "sync" + "syscall" "time" ) @@ -178,13 +178,43 @@ func whereFromEnv() (Where, error) { } // Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never -// share a dump. +// share a dump — across the night's process and the tool runtime alike (hold). type Backups struct { Where Where Run Runner Now func() time.Time Say func(format string, args ...any) - mu sync.Mutex +} + +// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a +// retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer +// keeps the other out. +const ( + nightLock = "night.lock" + measureLock = "measure.lock" +) + +// hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open +// file's, so two holders in one process exclude each other as two processes do. +func (b *Backups) hold(name string) (func(), error) { + f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err) + } + for { + err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX) + if !errors.Is(err, syscall.EINTR) { + break + } + } + if err != nil { + f.Close() + return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + f.Close() + }, nil } // exists is whether a path is there, asked as root: a store's directory is often its own user's @@ -299,16 +329,27 @@ func snapshotOf(out string) string { return id } -// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome. +// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and +// keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { - b.mu.Lock() - defer b.mu.Unlock() - if err := b.ensureRepository(ctx); err != nil { + run := b.LastRun() + run.At = b.Now().UTC() + failed := func(err error) (map[string]Night, error) { + run.Failed = err.Error() + b.keepRun(run) return nil, err } + release, err := b.hold(nightLock) + if err != nil { + return failed(err) + } + defer release() + if err := b.ensureRepository(ctx); err != nil { + return failed(err) + } all, err := b.Declared() if err != nil { - return nil, err + return failed(err) } chosen := all if only != "" { @@ -321,6 +362,7 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } } if len(chosen) == 0 { + // Asked of one module that is not here: nothing ran, and the last night stands as it was. return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names)) } } @@ -331,16 +373,59 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } // A retention that does not read thins nothing: keeping too much costs space, thinning by a value // nobody meant loses restore points (novox/hq issue 416). - keep, err := b.retention() + thinned := Thinning{At: b.Now().UTC()} + keep, err := b.retention(ctx) if err != nil { + thinned.Refused = err.Error() b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err) } else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", "--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil { + thinned.Kept, thinned.Error = &keep, err.Error() b.Say("thinning the restore points failed, and every one is kept: %v", err) + } else { + thinned.Kept, thinned.OK = &keep, true } + run.Failed, run.Thinned = "", &thinned + b.keepRun(run) return outcome, nil } +// Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its +// thinning. Each module's own outcome is its Night. +type Run struct { + At time.Time `json:"at"` + // Failed is why it did not run at all: the repository would not open, or what to back up does not read. + Failed string `json:"failed,omitempty"` + Thinned *Thinning `json:"thinned,omitempty"` +} + +// Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing. +type Thinning struct { + At time.Time `json:"at"` + OK bool `json:"ok"` + Kept *Retention `json:"kept,omitempty"` + Refused string `json:"refused,omitempty"` + Error string `json:"error,omitempty"` +} + +func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") } + +// LastRun is how the last night went as a whole; empty before the first. +func (b *Backups) LastRun() Run { + var r Run + if raw, err := os.ReadFile(b.runFile()); err == nil { + _ = json.Unmarshal(raw, &r) + } + return r +} + +func (b *Backups) keepRun(r Run) { + raw, _ := json.MarshalIndent(r, "", " ") + if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording how the night went failed: %v", err) + } +} + func orNothing(names []string) string { if len(names) == 0 { return "nothing" @@ -376,6 +461,11 @@ type ModuleBackups struct { LastNight *Night `json:"lastNight"` RestorePoints int `json:"restorePoints"` Newest *Snapshot `json:"newest,omitempty"` + // Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned + // how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its + // own, so the answer stays the list the controller reads. + Retention InForce `json:"retention"` + Thinned *Thinning `json:"thinned,omitempty"` // Data is every item the module declares, measured, with its newest good backup (novox/hq ADR // 0233): what the controller's self-check reads. Data []ItemReport `json:"data"` @@ -396,6 +486,8 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, good := b.Good() measured := b.Measurements() snaps, _ := b.Snapshots(ctx, module) + inForce := b.inForce(ctx) + thinned := b.LastRun().Thinned out := []ModuleBackups{} listed := map[string]bool{} for _, d := range declared { @@ -403,7 +495,7 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, continue } listed[d.Module] = true - m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, + m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned, Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])} if n, ok := nights[d.Module]; ok { m.LastNight = &n @@ -440,8 +532,11 @@ type Restored struct { // state as restored/// (besideIsUnsafe). A target that already exists is refused, never // overwritten. func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) { - b.mu.Lock() - defer b.mu.Unlock() + release, err := b.hold(nightLock) + if err != nil { + return nil, err + } + defer release() mine, err := b.Snapshots(ctx, module) if err != nil { return nil, err @@ -583,9 +678,12 @@ func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) { // Check is the weekly look at the repository's own integrity, with a sample of the data read back. func (b *Backups) Check(ctx context.Context) error { - b.mu.Lock() - defer b.mu.Unlock() - _, err := b.restic(ctx, "check", "--read-data-subset", "5%") + release, err := b.hold(nightLock) + if err != nil { + return err + } + defer release() + _, err = b.restic(ctx, "check", "--read-data-subset", "5%") return err } @@ -602,11 +700,6 @@ func nextNight(now time.Time, hour int) time.Time { // missedANight is whether a night was missed: the newest good night of any module is older than a // day and a bit — the machine was off, or this module was not running, at the hour. func missedANight(nights map[string]Night, now time.Time) bool { - var newest time.Time - for _, n := range nights { - if n.OK && n.At.After(newest) { - newest = n.At - } - } - return newest.IsZero() || now.Sub(newest) > 26*time.Hour + newest := newestGood(nights) + return newest.IsZero() || now.Sub(newest) > Overdue } diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go index 77b0dc1f..bf6ee69f 100644 --- a/modules/restic/cmd/restic-backups/backups_test.go +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -25,6 +25,8 @@ func placed(t *testing.T, declared string) Where { dir := t.TempDir() must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600)) must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600)) + // The retention file every machine has, as the mesh writes it with the defaults (novox/hq issue 454). + must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600)) return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"), PasswordFile: filepath.Join(dir, "pw"), State: dir} } diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index f0d6dd50..a7a3d5fc 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -157,8 +157,6 @@ func (b *Backups) Measurements() map[string]map[string]Measured { return out } -var measuring sync.Mutex - // The ways an item is measured. const ( measureWalk = "walk" @@ -364,8 +362,12 @@ func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } // measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the // hourly round reads counters, and a walk runs at most once a day. func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { - measuring.Lock() - defer measuring.Unlock() + // The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454). + release, err := b.hold(measureLock) + if err != nil { + return err + } + defer release() items, err := b.Items() if err != nil { return err @@ -552,8 +554,13 @@ func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm } func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) { - b.mu.Lock() - defer b.mu.Unlock() + release, err := b.hold(nightLock) + if err != nil { + d.Running, d.Done, d.Error = false, true, err.Error() + b.keepDeletion(path, d) + return + } + defer release() finish := func(err error) { d.Running, d.Done = false, true if err != nil { diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go new file mode 100644 index 00000000..1fd71ec0 --- /dev/null +++ b/modules/restic/cmd/restic-backups/health.go @@ -0,0 +1,189 @@ +package main + +// The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be +// loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines +// only — a typo in a setting let the repository grow until a disk condition fired. +// +// Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run +// at all, when no night has backed anything up while this machine was up for more than a day (awake.go), when +// the night's process has stopped counting, when the last thinning failed, or when the retention file is +// missing, cannot be read or is not in the form the mesh writes. A retention setting the night refuses is the operator's to give again: the +// answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator +// condition. The node-engine asks this through the node tools for the night's process; the controller +// raises module.restic..unhealthy on the second look that says so, with these words. + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why, +// waits), with what was found and what was said beside it. +type Health struct { + health.Answer + Found []string `json:"found,omitempty"` + Said []string `json:"said,omitempty"` +} + +// Seen is what the health is judged from. +type Seen struct { + Declared []Declared + DeclaredErr error + Nights map[string]Night + Good map[string]time.Time + Run Run + Retention InForce + // Awake is the night's process's count of how long this machine was up since its newest good night; nil + // before the process ever counted. + Awake *Awake + Now time.Time +} + +func hours(d time.Duration) string { + if d < 2*time.Hour { + return fmt.Sprintf("%d minutes", int(d.Minutes())) + } + return fmt.Sprintf("%d hours", int(d.Hours())) +} + +func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") } + +// Judge is the health of what was seen. Pure. +func Judge(s Seen) Health { + var found, said []string + var waits []health.Wait + if s.DeclaredErr != nil { + found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+ + "back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr)) + } + if s.Run.Failed != "" { + found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+ + "names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed)) + } + if a := s.Awake; a != nil { + newest := "no night has backed anything up here yet" + if !a.From.IsZero() { + newest = "the newest good night was at " + clock(a.From) + } + switch { + case s.Now.Sub(a.Looked) > Stale: + found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+ + "night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+ + "on this machine.", clock(a.Looked), hours(Stale), newest)) + case a.Up > Overdue: + found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+ + "night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+ + "the repository, or never starts leaves no failed night behind; read the journal of restic-night "+ + "(node-service-manager.journal), then take one with node-backup.now on this machine.", newest, hours(a.Up))) + default: + said = append(said, fmt.Sprintf("Up %s with the night's process running since the newest good night; "+ + "said when that passes %s.", hours(a.Up), hours(Overdue))) + } + } + failed := map[string]Night{} + for _, d := range s.Declared { + if n, ok := s.Nights[d.Module]; ok && !n.OK { + failed[d.Module] = n + } + } + names := make([]string, 0, len(failed)) + for m := range failed { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + n := failed[m] + since := "it has no restore point here" + if g, ok := s.Good[m]; ok && !g.IsZero() { + since = "its newest restore point is of " + clock(g) + } + found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+ + `with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m)) + } + if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" { + found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. Nothing was forgotten and every "+ + "restore point is kept, so the repository grows until a thinning succeeds; fix what the error names, then "+ + "node-backup.now on this machine thins again.", clock(t.At), strings.TrimSuffix(t.Error, "."))) + } + r := s.Retention + switch { + case r.Refused == "" && r.Retention != nil: + said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module.", + r.Daily, r.Weekly, r.Monthly)) + case r.Setting != "": + value := "not set" + if r.Value != "" { + value = fmt.Sprintf("%q", r.Value) + } + line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+ + "nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+ + "(mesh-controller.settings).", r.Setting, value, r.Setting) + said = append(said, line) + waits = append(waits, health.Wait{Part: "night", Setting: r.Setting, + What: "a count from 1 to 9999 for " + r.Setting}) + case strings.Contains(r.Refused, Unreadable): + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. The mesh writes the file readable (0644) and root's; one the account cannot "+ + "read was written by an older build of this module, or changed on the machine, and the next push of this "+ + "machine (mesh-controller.push) writes it readable again.", r.Refused)) + case strings.HasSuffix(r.Refused, Missing): + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept until it is there again, rather than thinned by a number nobody set; push this machine "+ + "(mesh-controller.push) so the mesh writes it again.", r.Refused)) + case strings.Contains(r.Refused, RetentionFile+": "): + // Read, and not in the form the mesh writes: something other than the mesh wrote it. + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. The mesh writes only daily=, weekly= and monthly=, so something else wrote "+ + "this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) + default: + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. Fix what the error names.", r.Refused)) + } + if len(found) > 0 { + why := strings.Join(found, " ") + if len(waits) > 0 { + why += " " + said[len(said)-1] + } + return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said} + } + if len(waits) > 0 { + return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said} + } + return Health{Answer: health.Healthy(healthySaid(s, said)), Said: said} +} + +// healthySaid is what was seen, and only that, when nothing is wrong. +func healthySaid(s Seen, said []string) string { + var parts []string + switch { + case len(s.Declared) == 0: + parts = append(parts, "No module here declares anything to back up.") + case newestGood(s.Nights).IsZero(): + parts = append(parts, "No good night is recorded here yet.") + default: + parts = append(parts, fmt.Sprintf("No module declared here failed its last night; the newest good night was at %s.", + clock(newestGood(s.Nights)))) + } + switch t := s.Run.Thinned; { + case t == nil: + parts = append(parts, "No thinning is recorded yet.") + case t.OK && t.Kept != nil: + parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points.", + clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly)) + case t.Refused != "": + parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At))) + } + return strings.Join(append(parts, said...), " ") +} + +// Health reads what the nights left and the retention as it is now, and judges them. +func (b *Backups) Health(ctx context.Context) Health { + declared, err := b.Declared() + return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), + Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()}) +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go new file mode 100644 index 00000000..452e4e4e --- /dev/null +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -0,0 +1,382 @@ +package main + +// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and +// a refused retention each raise a condition naming the module and what to do, instead of a journal line +// nobody reads until a disk fills. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails +// with thinning when it is not empty. +type nightly struct { + failing map[string]bool + thinning string +} + +func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]: + return "", errors.New("the dump failed: no space left on device") + case name == "restic" && args[5] == "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + case name == "restic" && args[5] == "forget" && n.thinning != "": + return "", errors.New(n.thinning) + } + return "", nil +} + +func nightAt(t *testing.T) (*Backups, *nightly) { + t.Helper() + n := &nightly{failing: map[string]bool{}} + b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run, + Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet} + return b, n +} + +func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) { + b, n := nightAt(t) + n.failing["mail"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a failed backup is %s: %s", h.State, h.Why) + } + for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } + if strings.Contains(h.Why, "pg was NOT") { + t.Errorf("pg was backed up and is said as failed: %q", h.Why) + } + + // Backed up again, it clears. + n.failing["mail"] = false + _, err = b.BackUp(context.Background(), "mail") + must(t, err) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("backed up again, it is %s: %s", h.State, h.Why) + } +} + +func TestANightThatDidNotRunIsUnhealthy(t *testing.T) { + b, _ := nightAt(t) + b.Run = func(_ context.Context, name string, args ...string) (string, error) { + return "", errors.New("Fatal: wrong password or no key found") + } + if _, err := b.BackUp(context.Background(), ""); err == nil { + t.Fatal("a repository that will not open ran a night") + } + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") { + t.Fatalf("a night that did not run: %s: %q", h.State, h.Why) + } +} + +func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) { + b, n := nightAt(t) + n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively" + _, err := b.BackUp(context.Background(), "") + must(t, err) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a failed thinning is %s: %s", h.State, h.Why) + } + for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } + n.thinning = "" + _, err = b.BackUp(context.Background(), "") + must(t, err) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("thinned again, it is %s: %s", h.State, h.Why) + } +} + +// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283), +// and the controller says it as the module's needs-operator condition with the act. +func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) { + b, _ := nightAt(t) + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600)) + h := b.Health(context.Background()) + if h.State != health.StateWaits { + t.Fatalf("a refused setting is %s: %s", h.State, h.Why) + } + if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil { + t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits) + } + for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } +} + +// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the +// operator gave: it is a fault, and says to push the machine. +func TestAMissingRetentionFileIsUnhealthy(t *testing.T) { + b, _ := nightAt(t) + must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile))) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a missing retention file is %s: %s", h.State, h.Why) + } + for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } +} + +// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for. +func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) { + b, n := nightAt(t) + n.failing["pg"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600)) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") { + t.Fatalf("%s: %q", h.State, h.Why) + } +} + +func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) { + b, _ := nightAt(t) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("before any night: %s: %s", h.State, h.Why) + } +} + +// A module no longer declared here is not held against the machine for a night it failed before. +func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) { + b, n := nightAt(t) + n.failing["mail"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600)) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why) + } +} + +// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning. +func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) { + b, n := nightAt(t) + n.thinning = "Fatal: repository is already locked" + _, err := b.BackUp(context.Background(), "") + must(t, err) + listed, err := b.BackedUp(context.Background(), "pg") + must(t, err) + if len(listed) != 1 { + t.Fatalf("listed %+v", listed) + } + raw, _ := json.Marshal(listed[0]) + var got struct { + Retention map[string]any `json:"retention"` + Thinned map[string]any `json:"thinned"` + } + must(t, json.Unmarshal(raw, &got)) + if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil { + t.Errorf("retention %v, want 14/8/6 not refused", got.Retention) + } + if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") { + t.Errorf("thinned %v, want the failure", got.Thinned) + } + + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600)) + listed, err = b.BackedUp(context.Background(), "pg") + must(t, err) + raw, _ = json.Marshal(listed[0]) + got.Retention = nil + must(t, json.Unmarshal(raw, &got)) + if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" { + t.Errorf("retention %v, want it refused naming keep-daily", got.Retention) + } +} + +// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at +// a time across both, never two nights or a night and a restore over one repository. +func TestANightWaitsForAnotherProcesssNight(t *testing.T) { + b, _ := nightAt(t) + release, err := b.hold(nightLock) + must(t, err) + done := make(chan struct{}) + go func() { + other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet} + _, _ = other.BackUp(context.Background(), "") + close(done) + }() + select { + case <-done: + t.Fatal("a night ran while another held the repository") + case <-time.After(200 * time.Millisecond): + } + release() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the night did not run once the other let go") + } +} + +// The manifest runs the night and the measuring as processes of their own, as the operator account like the +// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as +// before), each given where everything is; the night judged by backup_health, the measuring by its unit — +// a tool check only beside a check of another kind (ADR 0240 rule 2). +func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + must(t, err) + var m struct { + Tools []string `json:"tools"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []struct { + Name string `json:"name"` + Env map[string]string `json:"env"` + } `json:"artifacts"` + } `json:"build"` + } + must(t, json.Unmarshal(raw, &m)) + served := map[string]bool{} + for _, tool := range tools(&Backups{}) { + served[tool.Name] = true + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which the bundle does not serve", tool) + } + } + env := m.Build.Artifacts[0].Env + want := map[string]struct{ run, kind, tool string }{ + "night": {"night", "tool", "backup_health"}, + "measure": {"measure", "unit", ""}, + } + for _, r := range m.Resources { + w, ok := want[r["id"].(string)] + if !ok || r["type"] != "process" { + continue + } + delete(want, r["id"].(string)) + run, _ := json.Marshal(r["run"]) + if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name { + t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"]) + } + if r["user"] != "${machine:account}" { + t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"]) + } + penv, _ := r["env"].(map[string]any) + for k, v := range env { + if penv[k] != v { + t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v) + } + } + h, _ := r["health"].(map[string]any) + if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) { + t.Errorf("%s is judged by %v", r["id"], h) + } + if w.tool != "" && !slicesContains(m.Tools, w.tool) { + t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool) + } + } + for id := range want { + t.Errorf("no process %s", id) + } +} + +func slicesContains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a +// day since the newest good night, counted only while it was awake (novox/hq issue 454). +func TestANightThatNeverHappensIsUnhealthy(t *testing.T) { + good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, + Retention: InForce{Retention: &TodaysRetention}, Now: now} + + seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)} + h := Judge(seen) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") { + t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why) + } + + // A laptop asleep most of those two days: up 20 hours of them, nothing is overdue. + seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)} + if h := Judge(seen); h.State != health.StateHealthy { + t.Fatalf("up 20 hours: %s: %q", h.State, h.Why) + } + + // The process stopped counting: said, not taken for a night that is on time. + seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)} + if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { + t.Fatalf("a count an hour old: %s: %q", h.State, h.Why) + } +} + +func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) { + first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC) + a := Awake{}.Count(first, time.Minute, now) + a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake + if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) { + t.Fatalf("counted %+v", a) + } + a = a.Count(first.Add(24*time.Hour), time.Minute, now) + if a.Up != time.Minute { + t.Fatalf("a newer good night did not start the count again: %+v", a) + } +} + +// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one +// called a success. +func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) { + good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good} + h := Judge(seen) + if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") { + t.Fatalf("with no thinning recorded: %q", h.Why) + } + seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""} + if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") { + t.Fatalf("after a refused thinning: %q", h.Why) + } + seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24}} + if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") { + t.Fatalf("after a thinning: %q", h.Why) + } +} + +// The lock that cannot be taken is a night that did not run, and recorded as one. +func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) { + b, _ := nightAt(t) + must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write + if _, err := b.BackUp(context.Background(), ""); err == nil { + t.Fatal("a night ran without its lock") + } + if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") { + t.Fatalf("%s: %q", h.State, h.Why) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 508a69b4..2e9d4610 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -1,7 +1,16 @@ -// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the -// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It -// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the -// live data — and, beside them, runs the night that happens without anyone asking. +// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, three ways to run: +// +// - with no argument, launched by the machine's tool runtime and speaking MCP to it over stdio through the +// Go SDK (ADR 0193, ADR 0198): the node-backup seat's three verbs — what is backed up, take one now, +// restore beside the live data — and the health check, backup_health; +// - `night`, a process the node-engine runs: the night that happens without anyone asking, judged by +// backup_health from what it leaves; +// - `measure`, a process the node-engine runs: the hourly measuring of the data declared here (ADR 0233), +// judged by its unit. +// +// The night and the measuring were goroutines of the tool runtime's process until novox/hq issue 454: a module +// raises a condition only through the health of something it runs (ADR 0240, ADR 0256), and a check by the +// module's own tool only beside a check of another kind, so each runs as a process of its own. // // stdout is the MCP channel; everything this module says, it says on stderr. package main @@ -31,11 +40,24 @@ func main() { os.Exit(1) } b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say} - go nights(b) - go measurements(b) - if err := stdio.Serve("", tools(b)); err != nil { - say("%v", err) - os.Exit(1) + way := "" + if len(os.Args) > 1 { + way = os.Args[1] + } + switch way { + case "night": + nights(b) + case "measure": + measurements(b) + case "": + if err := stdio.Serve("", tools(b)); err != nil { + say("%v", err) + os.Exit(1) + } + default: + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+ + "measure it measures the data declared here", way) + os.Exit(2) } } @@ -47,6 +69,7 @@ func nights(b *Backups) { if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { hour = h } + go counting(b) if missedANight(b.Nights(), time.Now()) { time.Sleep(10 * time.Minute) night(b) @@ -154,6 +177,11 @@ func tools(b *Backups) []stdio.Tool { } return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path")) }), + {Name: "backup_health", Description: "Whether this machine's backups hold, as the module's health check " + + "(novox/hq ADR 0240, issue 454): unhealthy when a module was NOT backed up at its last night, the last night " + + "did not run, the last thinning failed, or the retention file is missing or unreadable — each said with what " + + "to do; waits, naming the setting, when a retention setting is refused. (r)", + Run: func(map[string]any) (any, error) { return b.Health(context.Background()), nil }}, // Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup // delete` asks these. Not seat verbs — nobody else calls them. {Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " + diff --git a/modules/restic/cmd/restic-backups/retention.go b/modules/restic/cmd/restic-backups/retention.go index 8c37ee0d..26d23bf7 100644 --- a/modules/restic/cmd/restic-backups/retention.go +++ b/modules/restic/cmd/restic-backups/retention.go @@ -7,8 +7,10 @@ package main // refused here and never handed to restic (issue 346: a raw setting is an injection point). import ( + "context" "errors" "fmt" + "io/fs" "os" "path/filepath" "regexp" @@ -17,10 +19,14 @@ import ( ) // Retention is how many restore points each period keeps. -type Retention struct{ Daily, Weekly, Monthly int } +type Retention struct { + Daily int `json:"daily"` + Weekly int `json:"weekly"` + Monthly int `json:"monthly"` +} // TodaysRetention is the rotation before it was a preference (novox/hq ADR 0214), and still the -// manifest's default. +// manifest's default — never a fallback for a file that is not there (novox/hq issue 454). var TodaysRetention = Retention{14, 8, 6} // RetentionFile is the file the mesh writes the retention into, in this module's state. @@ -52,32 +58,80 @@ func ParseRetention(text string) (Retention, error) { case seen[k] != 0: return Retention{}, fmt.Errorf("%s is given twice", k) case !count.MatchString(v): - return Retention{}, fmt.Errorf("the %s setting is %q; it is a count from 1 to 9999", setting, v) + return Retention{}, &SettingRefused{Setting: setting, Value: v} } seen[k], _ = strconv.Atoi(v) } for _, k := range []string{"daily", "weekly", "monthly"} { if seen[k] == 0 { - return Retention{}, fmt.Errorf("the %s setting is missing", settingOf[k]) + return Retention{}, &SettingRefused{Setting: settingOf[k]} } } return Retention{seen["daily"], seen["weekly"], seen["monthly"]}, nil } -// retention is this machine's: today's where the module was placed by a definition older than the -// file, else what the file says. -func (b *Backups) retention() (Retention, error) { +// SettingRefused is one retention setting the night refuses: the operator's to give again. +type SettingRefused struct{ Setting, Value string } + +func (e *SettingRefused) Error() string { + if e.Value == "" { + return fmt.Sprintf("the %s setting is missing", e.Setting) + } + return fmt.Sprintf("the %s setting is %q; it is a count from 1 to 9999", e.Setting, e.Value) +} + +// Missing is said of a retention file that is not there. Every machine has had the file since mesh-catalog +// #200 rolled out (seen in each machine's plan, 2026-10-11), so a missing one is lost, not old: it is refused +// rather than thinned by 14/8/6, which would thin by a number nobody set (novox/hq issue 454). +const Missing = "is missing" + +// Unreadable is said of a retention file this program's account may not read. +const Unreadable = "cannot be read by the account this module runs as" + +// retention is this machine's, as the file the mesh wrote says; an error when it is missing, cannot be +// read, or does not read, and a *SettingRefused within it when one setting is what is wrong. +// +// The mesh writes the file root's and readable by everyone (0644): the account the night runs as reads it +// and cannot change it, and the counts are no secret (novox/hq issue 454 — written 0600 before, which the +// account could not read, so every night would have refused it and thinned nothing). +func (b *Backups) retention(context.Context) (Retention, error) { path := filepath.Join(b.Where.State, RetentionFile) raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return TodaysRetention, nil - } - if err != nil { + switch { + case errors.Is(err, fs.ErrNotExist): + return Retention{}, fmt.Errorf("%s %s", path, Missing) + case errors.Is(err, fs.ErrPermission): + return Retention{}, fmt.Errorf("%s %s (%v)", path, Unreadable, errors.Unwrap(err)) + case err != nil: return Retention{}, err } - r, err := ParseRetention(string(raw)) + keep, err := ParseRetention(string(raw)) if err != nil { return Retention{}, fmt.Errorf("%s: %w", path, err) } - return r, nil + return keep, nil +} + +// InForce is the retention a night thins by, or why it thins nothing: what `backed-up` says beside each +// module and what the health judges. +type InForce struct { + *Retention + Refused string `json:"refused,omitempty"` + // Setting is the setting a refusal names, when one setting is what is wrong. + Setting string `json:"setting,omitempty"` + // Value is what that setting is now. + Value string `json:"-"` +} + +func (b *Backups) inForce(ctx context.Context) InForce { + keep, err := b.retention(ctx) + if err == nil { + return InForce{Retention: &keep} + } + f := InForce{Refused: err.Error()} + var s *SettingRefused + if errors.As(err, &s) { + f.Setting, f.Value = s.Setting, s.Value + } + return f } diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index 4969d690..9b489b80 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -21,8 +21,11 @@ import ( func thinning(t *testing.T, retention *string) (string, []string) { t.Helper() where := placed(t, "# pg\npath /\n") + path := filepath.Join(where.State, RetentionFile) if retention != nil { - must(t, os.WriteFile(filepath.Join(where.State, "retention.conf"), []byte(*retention), 0o600)) + must(t, os.WriteFile(path, []byte(*retention), 0o600)) + } else { + must(t, os.Remove(path)) } forget := "" run := func(_ context.Context, name string, args ...string) (string, error) { @@ -53,11 +56,76 @@ func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) { } } -// A machine placed by a definition older than the file keeps today's rotation. -func TestWithoutTheFileANightKeepsTodaysRetention(t *testing.T) { - got, _ := thinning(t, nil) - if want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6"; got != want { - t.Errorf("ran %q, want %q", got, want) +// Every machine has had the file since mesh-catalog #200 rolled out (novox/hq issue 454), so a missing one is +// a fault, never a reason to thin by 14/8/6: a lost file would otherwise thin by a number nobody set. +func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) { + got, said := thinning(t, nil) + if got != "" { + t.Errorf("ran %q without a retention file", got) + } + all := strings.Join(said, "\n") + if !strings.Contains(all, "retention.conf is missing") || !strings.Contains(all, "every restore point is kept") { + t.Errorf("said %q, want it to say the file is missing and every restore point is kept", all) + } +} + +// A file the account the night runs as may not read — the mode an older build of this module had the mesh +// write it in (0600, root's) — thins nothing, and says why (novox/hq issue 454). +func TestARetentionTheAccountCannotReadThinsNothingAndSaysWhy(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root reads a file of mode 0 itself") + } + where := placed(t, "# pg\npath /\n") + must(t, os.Chmod(filepath.Join(where.State, RetentionFile), 0)) + forget := "" + var said []string + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "restic" && args[5] == "forget": + forget = "restic " + strings.Join(args[5:], " ") + case name == "restic" && args[5] == "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + } + return "", nil + } + b := &Backups{Where: where, Run: run, Now: time.Now, Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + _, err := b.BackUp(context.Background(), "") + must(t, err) + if forget != "" { + t.Errorf("thinned by a file it could not read: %q", forget) + } + if all := strings.Join(said, "\n"); !strings.Contains(all, Unreadable) || !strings.Contains(all, "every restore point is kept") { + t.Errorf("said %q", all) + } + h := b.Health(context.Background()) + if h.State != "unhealthy" || !strings.Contains(h.Why, Unreadable) || !strings.Contains(h.Why, "0644") { + t.Errorf("%s: %q, want it unhealthy naming the cause", h.State, h.Why) + } +} + +// The mesh writes the file readable, and root's: the account reads it and cannot change it. +func TestTheManifestWritesTheRetentionReadable(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + must(t, err) + var m struct { + Resources []map[string]any `json:"resources"` + } + must(t, json.Unmarshal(raw, &m)) + file, night := -1, -1 + for i, r := range m.Resources { + switch r["id"] { + case "retention": + file = i + if r["mode"] != "0644" || r["owner"] != nil { + t.Errorf("the retention file is mode %v owner %v, want 0644 and root's", r["mode"], r["owner"]) + } + case "night": + night = i + } + } + // Applied in declared order: the rollout that brings the night's process makes the file readable first. + if file < 0 || night < 0 || file > night { + t.Errorf("the retention file is declared at %d and the night at %d", file, night) } } diff --git a/modules/restic/go.mod b/modules/restic/go.mod index f010f2fc..7ec16d84 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -2,4 +2,4 @@ module restic go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c diff --git a/modules/restic/go.sum b/modules/restic/go.sum index b474419e..dfc73823 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/restic/module.json b/modules/restic/module.json index da5af259..544c0b81 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -12,6 +12,11 @@ ] } ], + "tools": [ + "backup_health", + "backup_delete_retired", + "backup_deleted" + ], "own-secrets": { "repository": "${dir:state}/repository.secret" }, @@ -79,7 +84,7 @@ "id": "retention", "type": "file", "path": "${dir:state}/retention.conf", - "mode": "0600", + "mode": "0644", "trusted": true, "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly and keep-monthly (novox/hq issue 416). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\n" }, @@ -92,6 +97,53 @@ "id": "sqlite", "type": "package", "package": "sqlite" + }, + { + "id": "night", + "type": "process", + "name": "restic-night", + "artifact": "tools", + "run": [ + "./restic-backups", + "night" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "measure", + "type": "process", + "name": "restic-measure", + "artifact": "tools", + "run": [ + "./restic-backups", + "measure" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "unit" + } } ], "build": {