From d649309895b0f8a3e4b6b7235d3d2f6feaed4045 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:56:23 +0200 Subject: [PATCH 1/2] restic: raise a failed backup, a failed thinning and a refused retention as the module's health (novox/hq issue 454) A failed night was a journal line only, so a typo in a retention setting let the repository grow until a disk filled. The night and the measuring now run as processes the node-engine judges: the night by backup_health, which is unhealthy for a module not backed up, a night that did not run, a failed thinning or a missing retention file, and waits naming the setting when one is refused; the measuring by its unit. A file lock keeps a night and the tool runtime's now and restore apart. The retention file is root's alone, so the account reads it through sudo: until now every night would have refused it as unreadable. Every machine has the file, so a missing one is refused instead of thinned by 14/8/6. backed-up says the retention in force and the last thinning per module. --- modules/restic/cmd/restic-backups/backups.go | 135 +++++++- .../restic/cmd/restic-backups/backups_test.go | 2 + modules/restic/cmd/restic-backups/data.go | 19 +- modules/restic/cmd/restic-backups/health.go | 133 ++++++++ .../restic/cmd/restic-backups/health_test.go | 308 ++++++++++++++++++ modules/restic/cmd/restic-backups/main.go | 45 ++- .../restic/cmd/restic-backups/retention.go | 108 +++++- .../cmd/restic-backups/retention_test.go | 55 +++- modules/restic/go.mod | 2 +- modules/restic/go.sum | 4 +- modules/restic/module.json | 52 +++ 11 files changed, 807 insertions(+), 56 deletions(-) create mode 100644 modules/restic/cmd/restic-backups/health.go create mode 100644 modules/restic/cmd/restic-backups/health_test.go diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 3b67e781..d6439e10 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -9,9 +9,9 @@ // // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and // restored on its own. Everything lands in one repository on the machine — deduplicated, so every -// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, -// 8 weekly and 6 monthly unless the machine's settings say otherwise (retention.go). Against -// mistakes, not disasters: nothing leaves the machine. +// night is a complete restore point and only what changed costs space — and is thinned by the machine's +// settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters: +// nothing leaves the machine. How the last night and thinning went is the module's health (health.go). // // Root's: the dumps read every store and the repository holds every module's data, and the runtime // launching this binary runs as the operator's account, so restic and the run lines go through sudo @@ -32,7 +32,7 @@ import ( "slices" "strconv" "strings" - "sync" + "syscall" "time" ) @@ -178,13 +178,44 @@ func whereFromEnv() (Where, error) { } // Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never -// share a dump. +// share a dump — across the night's process and the tool runtime alike (hold). type Backups struct { Where Where Run Runner Now func() time.Time Say func(format string, args ...any) - mu sync.Mutex + read retentionRead +} + +// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a +// retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer +// keeps the other out. +const ( + nightLock = "night.lock" + measureLock = "measure.lock" +) + +// hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open +// file's, so two holders in one process exclude each other as two processes do. +func (b *Backups) hold(name string) (func(), error) { + f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err) + } + for { + err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX) + if !errors.Is(err, syscall.EINTR) { + break + } + } + if err != nil { + f.Close() + return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + f.Close() + }, nil } // exists is whether a path is there, asked as root: a store's directory is often its own user's @@ -299,16 +330,27 @@ func snapshotOf(out string) string { return id } -// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome. +// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and +// keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { - b.mu.Lock() - defer b.mu.Unlock() - if err := b.ensureRepository(ctx); err != nil { + release, err := b.hold(nightLock) + if err != nil { return nil, err } + defer release() + run := b.LastRun() + run.At = b.Now().UTC() + failed := func(err error) (map[string]Night, error) { + run.Failed = err.Error() + b.keepRun(run) + return nil, err + } + if err := b.ensureRepository(ctx); err != nil { + return failed(err) + } all, err := b.Declared() if err != nil { - return nil, err + return failed(err) } chosen := all if only != "" { @@ -321,6 +363,7 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } } if len(chosen) == 0 { + // Asked of one module that is not here: nothing ran, and the last night stands as it was. return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names)) } } @@ -331,16 +374,59 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } // A retention that does not read thins nothing: keeping too much costs space, thinning by a value // nobody meant loses restore points (novox/hq issue 416). - keep, err := b.retention() + thinned := Thinning{At: b.Now().UTC()} + keep, err := b.retention(ctx) if err != nil { + thinned.Refused = err.Error() b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err) } else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", "--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil { + thinned.Kept, thinned.Error = &keep, err.Error() b.Say("thinning the restore points failed, and every one is kept: %v", err) + } else { + thinned.Kept, thinned.OK = &keep, true } + run.Failed, run.Thinned = "", &thinned + b.keepRun(run) return outcome, nil } +// Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its +// thinning. Each module's own outcome is its Night. +type Run struct { + At time.Time `json:"at"` + // Failed is why it did not run at all: the repository would not open, or what to back up does not read. + Failed string `json:"failed,omitempty"` + Thinned *Thinning `json:"thinned,omitempty"` +} + +// Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing. +type Thinning struct { + At time.Time `json:"at"` + OK bool `json:"ok"` + Kept *Retention `json:"kept,omitempty"` + Refused string `json:"refused,omitempty"` + Error string `json:"error,omitempty"` +} + +func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") } + +// LastRun is how the last night went as a whole; empty before the first. +func (b *Backups) LastRun() Run { + var r Run + if raw, err := os.ReadFile(b.runFile()); err == nil { + _ = json.Unmarshal(raw, &r) + } + return r +} + +func (b *Backups) keepRun(r Run) { + raw, _ := json.MarshalIndent(r, "", " ") + if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording how the night went failed: %v", err) + } +} + func orNothing(names []string) string { if len(names) == 0 { return "nothing" @@ -376,6 +462,11 @@ type ModuleBackups struct { LastNight *Night `json:"lastNight"` RestorePoints int `json:"restorePoints"` Newest *Snapshot `json:"newest,omitempty"` + // Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned + // how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its + // own, so the answer stays the list the controller reads. + Retention InForce `json:"retention"` + Thinned *Thinning `json:"thinned,omitempty"` // Data is every item the module declares, measured, with its newest good backup (novox/hq ADR // 0233): what the controller's self-check reads. Data []ItemReport `json:"data"` @@ -396,6 +487,8 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, good := b.Good() measured := b.Measurements() snaps, _ := b.Snapshots(ctx, module) + inForce := b.inForce(ctx) + thinned := b.LastRun().Thinned out := []ModuleBackups{} listed := map[string]bool{} for _, d := range declared { @@ -403,7 +496,7 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, continue } listed[d.Module] = true - m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, + m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned, Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])} if n, ok := nights[d.Module]; ok { m.LastNight = &n @@ -440,8 +533,11 @@ type Restored struct { // state as restored/// (besideIsUnsafe). A target that already exists is refused, never // overwritten. func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) { - b.mu.Lock() - defer b.mu.Unlock() + release, err := b.hold(nightLock) + if err != nil { + return nil, err + } + defer release() mine, err := b.Snapshots(ctx, module) if err != nil { return nil, err @@ -583,9 +679,12 @@ func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) { // Check is the weekly look at the repository's own integrity, with a sample of the data read back. func (b *Backups) Check(ctx context.Context) error { - b.mu.Lock() - defer b.mu.Unlock() - _, err := b.restic(ctx, "check", "--read-data-subset", "5%") + release, err := b.hold(nightLock) + if err != nil { + return err + } + defer release() + _, err = b.restic(ctx, "check", "--read-data-subset", "5%") return err } diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go index 77b0dc1f..bf6ee69f 100644 --- a/modules/restic/cmd/restic-backups/backups_test.go +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -25,6 +25,8 @@ func placed(t *testing.T, declared string) Where { dir := t.TempDir() must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600)) must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600)) + // The retention file every machine has, as the mesh writes it with the defaults (novox/hq issue 454). + must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600)) return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"), PasswordFile: filepath.Join(dir, "pw"), State: dir} } diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index f0d6dd50..a7a3d5fc 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -157,8 +157,6 @@ func (b *Backups) Measurements() map[string]map[string]Measured { return out } -var measuring sync.Mutex - // The ways an item is measured. const ( measureWalk = "walk" @@ -364,8 +362,12 @@ func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } // measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the // hourly round reads counters, and a walk runs at most once a day. func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { - measuring.Lock() - defer measuring.Unlock() + // The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454). + release, err := b.hold(measureLock) + if err != nil { + return err + } + defer release() items, err := b.Items() if err != nil { return err @@ -552,8 +554,13 @@ func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm } func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) { - b.mu.Lock() - defer b.mu.Unlock() + release, err := b.hold(nightLock) + if err != nil { + d.Running, d.Done, d.Error = false, true, err.Error() + b.keepDeletion(path, d) + return + } + defer release() finish := func(err error) { d.Running, d.Done = false, true if err != nil { diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go new file mode 100644 index 00000000..c5be2466 --- /dev/null +++ b/modules/restic/cmd/restic-backups/health.go @@ -0,0 +1,133 @@ +package main + +// The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be +// loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines +// only — a typo in a setting let the repository grow until a disk condition fired. +// +// Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run +// at all, when the last thinning failed, or when the retention file is missing, cannot be read or is not in +// the form the mesh writes. A retention setting the night refuses is the operator's to give again: the +// answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator +// condition. The node-engine asks this through the node tools for the night's process; the controller +// raises module.restic..unhealthy on the second look that says so, with these words. + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why, +// waits), with what was found and what was said beside it. +type Health struct { + health.Answer + Found []string `json:"found,omitempty"` + Said []string `json:"said,omitempty"` +} + +// Seen is what the health is judged from. +type Seen struct { + Declared []Declared + DeclaredErr error + Nights map[string]Night + Good map[string]time.Time + Run Run + Retention InForce +} + +func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") } + +// Judge is the health of what was seen. Pure. +func Judge(s Seen) Health { + var found, said []string + var waits []health.Wait + if s.DeclaredErr != nil { + found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+ + "back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr)) + } + if s.Run.Failed != "" { + found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+ + "names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed)) + } + failed := map[string]Night{} + for _, d := range s.Declared { + if n, ok := s.Nights[d.Module]; ok && !n.OK { + failed[d.Module] = n + } + } + names := make([]string, 0, len(failed)) + for m := range failed { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + n := failed[m] + since := "it has no restore point here" + if g, ok := s.Good[m]; ok && !g.IsZero() { + since = "its newest restore point is of " + clock(g) + } + found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+ + `with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m)) + } + if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" { + found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. Nothing was forgotten and every "+ + "restore point is kept, so the repository grows until a thinning succeeds; fix what the error names, then "+ + "node-backup.now on this machine thins again.", clock(t.At), strings.TrimSuffix(t.Error, "."))) + } + r := s.Retention + switch { + case r.Refused == "" && r.Retention != nil: + said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module.", + r.Daily, r.Weekly, r.Monthly)) + case r.Setting != "": + value := "not set" + if r.Value != "" { + value = fmt.Sprintf("%q", r.Value) + } + line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+ + "nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+ + "(mesh-controller.settings).", r.Setting, value, r.Setting) + said = append(said, line) + waits = append(waits, health.Wait{Part: "night", Setting: r.Setting, + What: "a count from 1 to 9999 for " + r.Setting}) + case strings.HasSuffix(r.Refused, Missing): + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept until it is there again, rather than thinned by a number nobody set; push this machine "+ + "(mesh-controller.push) so the mesh writes it again.", r.Refused)) + default: + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) + } + if len(found) > 0 { + why := strings.Join(found, " ") + if len(waits) > 0 { + why += " " + said[len(said)-1] + } + return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said} + } + if len(waits) > 0 { + return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said} + } + why := "Every module declared here was backed up at its last night, and the last thinning succeeded." + switch { + case len(s.Declared) == 0: + why = "No module here declares anything to back up." + case len(s.Nights) == 0: + why = "No night has run here yet." + } + if len(said) > 0 { + why += " " + said[0] + } + return Health{Answer: health.Healthy(why), Said: said} +} + +// Health reads what the nights left and the retention as it is now, and judges them. +func (b *Backups) Health(ctx context.Context) Health { + declared, err := b.Declared() + return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), + Retention: b.inForce(ctx)}) +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go new file mode 100644 index 00000000..f0c97115 --- /dev/null +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -0,0 +1,308 @@ +package main + +// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and +// a refused retention each raise a condition naming the module and what to do, instead of a journal line +// nobody reads until a disk fills. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails +// with thinning when it is not empty. +type nightly struct { + failing map[string]bool + thinning string +} + +func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]: + return "", errors.New("the dump failed: no space left on device") + case name == "restic" && args[5] == "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + case name == "restic" && args[5] == "forget" && n.thinning != "": + return "", errors.New(n.thinning) + } + return "", nil +} + +func nightAt(t *testing.T) (*Backups, *nightly) { + t.Helper() + n := &nightly{failing: map[string]bool{}} + b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run, + Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet} + return b, n +} + +func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) { + b, n := nightAt(t) + n.failing["mail"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a failed backup is %s: %s", h.State, h.Why) + } + for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } + if strings.Contains(h.Why, "pg was NOT") { + t.Errorf("pg was backed up and is said as failed: %q", h.Why) + } + + // Backed up again, it clears. + n.failing["mail"] = false + _, err = b.BackUp(context.Background(), "mail") + must(t, err) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("backed up again, it is %s: %s", h.State, h.Why) + } +} + +func TestANightThatDidNotRunIsUnhealthy(t *testing.T) { + b, _ := nightAt(t) + b.Run = func(_ context.Context, name string, args ...string) (string, error) { + return "", errors.New("Fatal: wrong password or no key found") + } + if _, err := b.BackUp(context.Background(), ""); err == nil { + t.Fatal("a repository that will not open ran a night") + } + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") { + t.Fatalf("a night that did not run: %s: %q", h.State, h.Why) + } +} + +func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) { + b, n := nightAt(t) + n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively" + _, err := b.BackUp(context.Background(), "") + must(t, err) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a failed thinning is %s: %s", h.State, h.Why) + } + for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } + n.thinning = "" + _, err = b.BackUp(context.Background(), "") + must(t, err) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("thinned again, it is %s: %s", h.State, h.Why) + } +} + +// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283), +// and the controller says it as the module's needs-operator condition with the act. +func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) { + b, _ := nightAt(t) + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600)) + h := b.Health(context.Background()) + if h.State != health.StateWaits { + t.Fatalf("a refused setting is %s: %s", h.State, h.Why) + } + if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil { + t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits) + } + for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } +} + +// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the +// operator gave: it is a fault, and says to push the machine. +func TestAMissingRetentionFileIsUnhealthy(t *testing.T) { + b, _ := nightAt(t) + must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile))) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a missing retention file is %s: %s", h.State, h.Why) + } + for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } +} + +// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for. +func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) { + b, n := nightAt(t) + n.failing["pg"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600)) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") { + t.Fatalf("%s: %q", h.State, h.Why) + } +} + +func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) { + b, _ := nightAt(t) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("before any night: %s: %s", h.State, h.Why) + } +} + +// A module no longer declared here is not held against the machine for a night it failed before. +func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) { + b, n := nightAt(t) + n.failing["mail"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600)) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why) + } +} + +// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning. +func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) { + b, n := nightAt(t) + n.thinning = "Fatal: repository is already locked" + _, err := b.BackUp(context.Background(), "") + must(t, err) + listed, err := b.BackedUp(context.Background(), "pg") + must(t, err) + if len(listed) != 1 { + t.Fatalf("listed %+v", listed) + } + raw, _ := json.Marshal(listed[0]) + var got struct { + Retention map[string]any `json:"retention"` + Thinned map[string]any `json:"thinned"` + } + must(t, json.Unmarshal(raw, &got)) + if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil { + t.Errorf("retention %v, want 14/8/6 not refused", got.Retention) + } + if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") { + t.Errorf("thinned %v, want the failure", got.Thinned) + } + + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600)) + listed, err = b.BackedUp(context.Background(), "pg") + must(t, err) + raw, _ = json.Marshal(listed[0]) + got.Retention = nil + must(t, json.Unmarshal(raw, &got)) + if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" { + t.Errorf("retention %v, want it refused naming keep-daily", got.Retention) + } +} + +// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at +// a time across both, never two nights or a night and a restore over one repository. +func TestANightWaitsForAnotherProcesssNight(t *testing.T) { + b, _ := nightAt(t) + release, err := b.hold(nightLock) + must(t, err) + done := make(chan struct{}) + go func() { + other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet} + _, _ = other.BackUp(context.Background(), "") + close(done) + }() + select { + case <-done: + t.Fatal("a night ran while another held the repository") + case <-time.After(200 * time.Millisecond): + } + release() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the night did not run once the other let go") + } +} + +// The manifest runs the night and the measuring as processes of their own, as the operator account like the +// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as +// before), each given where everything is; the night judged by backup_health, the measuring by its unit — +// a tool check only beside a check of another kind (ADR 0240 rule 2). +func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + must(t, err) + var m struct { + Tools []string `json:"tools"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []struct { + Name string `json:"name"` + Env map[string]string `json:"env"` + } `json:"artifacts"` + } `json:"build"` + } + must(t, json.Unmarshal(raw, &m)) + served := map[string]bool{} + for _, tool := range tools(&Backups{}) { + served[tool.Name] = true + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which the bundle does not serve", tool) + } + } + env := m.Build.Artifacts[0].Env + want := map[string]struct{ run, kind, tool string }{ + "night": {"night", "tool", "backup_health"}, + "measure": {"measure", "unit", ""}, + } + for _, r := range m.Resources { + w, ok := want[r["id"].(string)] + if !ok || r["type"] != "process" { + continue + } + delete(want, r["id"].(string)) + run, _ := json.Marshal(r["run"]) + if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name { + t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"]) + } + if r["user"] != "${machine:account}" { + t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"]) + } + penv, _ := r["env"].(map[string]any) + for k, v := range env { + if penv[k] != v { + t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v) + } + } + h, _ := r["health"].(map[string]any) + if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) { + t.Errorf("%s is judged by %v", r["id"], h) + } + if w.tool != "" && !slicesContains(m.Tools, w.tool) { + t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool) + } + } + for id := range want { + t.Errorf("no process %s", id) + } +} + +func slicesContains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 508a69b4..648b9d49 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -1,7 +1,16 @@ -// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the -// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It -// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the -// live data — and, beside them, runs the night that happens without anyone asking. +// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, three ways to run: +// +// - with no argument, launched by the machine's tool runtime and speaking MCP to it over stdio through the +// Go SDK (ADR 0193, ADR 0198): the node-backup seat's three verbs — what is backed up, take one now, +// restore beside the live data — and the health check, backup_health; +// - `night`, a process the node-engine runs: the night that happens without anyone asking, judged by +// backup_health from what it leaves; +// - `measure`, a process the node-engine runs: the hourly measuring of the data declared here (ADR 0233), +// judged by its unit. +// +// The night and the measuring were goroutines of the tool runtime's process until novox/hq issue 454: a module +// raises a condition only through the health of something it runs (ADR 0240, ADR 0256), and a check by the +// module's own tool only beside a check of another kind, so each runs as a process of its own. // // stdout is the MCP channel; everything this module says, it says on stderr. package main @@ -31,11 +40,24 @@ func main() { os.Exit(1) } b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say} - go nights(b) - go measurements(b) - if err := stdio.Serve("", tools(b)); err != nil { - say("%v", err) - os.Exit(1) + way := "" + if len(os.Args) > 1 { + way = os.Args[1] + } + switch way { + case "night": + nights(b) + case "measure": + measurements(b) + case "": + if err := stdio.Serve("", tools(b)); err != nil { + say("%v", err) + os.Exit(1) + } + default: + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+ + "measure it measures the data declared here", way) + os.Exit(2) } } @@ -154,6 +176,11 @@ func tools(b *Backups) []stdio.Tool { } return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path")) }), + {Name: "backup_health", Description: "Whether this machine's backups hold, as the module's health check " + + "(novox/hq ADR 0240, issue 454): unhealthy when a module was NOT backed up at its last night, the last night " + + "did not run, the last thinning failed, or the retention file is missing or unreadable — each said with what " + + "to do; waits, naming the setting, when a retention setting is refused. (r)", + Run: func(map[string]any) (any, error) { return b.Health(context.Background()), nil }}, // Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup // delete` asks these. Not seat verbs — nobody else calls them. {Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " + diff --git a/modules/restic/cmd/restic-backups/retention.go b/modules/restic/cmd/restic-backups/retention.go index 8c37ee0d..90da6b1c 100644 --- a/modules/restic/cmd/restic-backups/retention.go +++ b/modules/restic/cmd/restic-backups/retention.go @@ -7,20 +7,28 @@ package main // refused here and never handed to restic (issue 346: a raw setting is an injection point). import ( + "context" "errors" "fmt" + "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" + "sync" + "syscall" ) // Retention is how many restore points each period keeps. -type Retention struct{ Daily, Weekly, Monthly int } +type Retention struct { + Daily int `json:"daily"` + Weekly int `json:"weekly"` + Monthly int `json:"monthly"` +} // TodaysRetention is the rotation before it was a preference (novox/hq ADR 0214), and still the -// manifest's default. +// manifest's default — never a fallback for a file that is not there (novox/hq issue 454). var TodaysRetention = Retention{14, 8, 6} // RetentionFile is the file the mesh writes the retention into, in this module's state. @@ -52,32 +60,104 @@ func ParseRetention(text string) (Retention, error) { case seen[k] != 0: return Retention{}, fmt.Errorf("%s is given twice", k) case !count.MatchString(v): - return Retention{}, fmt.Errorf("the %s setting is %q; it is a count from 1 to 9999", setting, v) + return Retention{}, &SettingRefused{Setting: setting, Value: v} } seen[k], _ = strconv.Atoi(v) } for _, k := range []string{"daily", "weekly", "monthly"} { if seen[k] == 0 { - return Retention{}, fmt.Errorf("the %s setting is missing", settingOf[k]) + return Retention{}, &SettingRefused{Setting: settingOf[k]} } } return Retention{seen["daily"], seen["weekly"], seen["monthly"]}, nil } -// retention is this machine's: today's where the module was placed by a definition older than the -// file, else what the file says. -func (b *Backups) retention() (Retention, error) { +// SettingRefused is one retention setting the night refuses: the operator's to give again. +type SettingRefused struct{ Setting, Value string } + +func (e *SettingRefused) Error() string { + if e.Value == "" { + return fmt.Sprintf("the %s setting is missing", e.Setting) + } + return fmt.Sprintf("the %s setting is %q; it is a count from 1 to 9999", e.Setting, e.Value) +} + +// Missing is said of a retention file that is not there. Every machine has had the file since mesh-catalog +// #200 rolled out (seen in each machine's plan, 2026-10-11), so a missing one is lost, not old: it is refused +// rather than thinned by 14/8/6, which would thin by a number nobody set (novox/hq issue 454). +const Missing = "is missing" + +// retentionRead is the last read of the file, by what its stat said, so a health look every minute does not +// run sudo every minute while nothing changed. +type retentionRead struct { + mu sync.Mutex + key string + keep Retention + err error +} + +// retention is this machine's, as the file the mesh wrote says; an error when it is missing, cannot be +// read, or does not read, and a *SettingRefused within it when one setting is what is wrong. +// +// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339) and this program runs as the +// operator account, so what the account may not read is read as root, as the declared directories are +// (novox/hq issue 454: until then every night refused it as unreadable and thinned nothing). +func (b *Backups) retention(ctx context.Context) (Retention, error) { path := filepath.Join(b.Where.State, RetentionFile) - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return TodaysRetention, nil + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return Retention{}, fmt.Errorf("%s %s", path, Missing) } if err != nil { return Retention{}, err } - r, err := ParseRetention(string(raw)) - if err != nil { - return Retention{}, fmt.Errorf("%s: %w", path, err) + key := fmt.Sprint(info.ModTime().UnixNano(), info.Size(), info.Mode()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + key += fmt.Sprint(" ", st.Ino, st.Ctim.Nano()) } - return r, nil + b.read.mu.Lock() + defer b.read.mu.Unlock() + if b.read.key == key { + return b.read.keep, b.read.err + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrPermission) { + var out string + out, err = b.Run(ctx, "cat", "--", path) + raw = []byte(out) + } + if err != nil { + // Not kept: a read that failed may succeed at the next look. + return Retention{}, fmt.Errorf("%s cannot be read: %w", path, err) + } + keep, err := ParseRetention(string(raw)) + if err != nil { + err = fmt.Errorf("%s: %w", path, err) + } + b.read.key, b.read.keep, b.read.err = key, keep, err + return keep, err +} + +// InForce is the retention a night thins by, or why it thins nothing: what `backed-up` says beside each +// module and what the health judges. +type InForce struct { + *Retention + Refused string `json:"refused,omitempty"` + // Setting is the setting a refusal names, when one setting is what is wrong. + Setting string `json:"setting,omitempty"` + // Value is what that setting is now. + Value string `json:"-"` +} + +func (b *Backups) inForce(ctx context.Context) InForce { + keep, err := b.retention(ctx) + if err == nil { + return InForce{Retention: &keep} + } + f := InForce{Refused: err.Error()} + var s *SettingRefused + if errors.As(err, &s) { + f.Setting, f.Value = s.Setting, s.Value + } + return f } diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index 4969d690..7449ba30 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -21,8 +21,11 @@ import ( func thinning(t *testing.T, retention *string) (string, []string) { t.Helper() where := placed(t, "# pg\npath /\n") + path := filepath.Join(where.State, RetentionFile) if retention != nil { - must(t, os.WriteFile(filepath.Join(where.State, "retention.conf"), []byte(*retention), 0o600)) + must(t, os.WriteFile(path, []byte(*retention), 0o600)) + } else { + must(t, os.Remove(path)) } forget := "" run := func(_ context.Context, name string, args ...string) (string, error) { @@ -53,11 +56,51 @@ func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) { } } -// A machine placed by a definition older than the file keeps today's rotation. -func TestWithoutTheFileANightKeepsTodaysRetention(t *testing.T) { - got, _ := thinning(t, nil) - if want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6"; got != want { - t.Errorf("ran %q, want %q", got, want) +// Every machine has had the file since mesh-catalog #200 rolled out (novox/hq issue 454), so a missing one is +// a fault, never a reason to thin by 14/8/6: a lost file would otherwise thin by a number nobody set. +func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) { + got, said := thinning(t, nil) + if got != "" { + t.Errorf("ran %q without a retention file", got) + } + all := strings.Join(said, "\n") + if !strings.Contains(all, "retention.conf is missing") || !strings.Contains(all, "every restore point is kept") { + t.Errorf("said %q, want it to say the file is missing and every restore point is kept", all) + } +} + +// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339), and the night runs as the +// operator account: what the account cannot read is read as root, as every other root's file here is +// (novox/hq issue 454). Before, every machine's night refused it as unreadable and thinned nothing. +func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) { + where := placed(t, "# pg\npath /\n") + path := filepath.Join(where.State, RetentionFile) + must(t, os.WriteFile(path, []byte("daily=7\nweekly=4\nmonthly=24\n"), 0o600)) + must(t, os.Chmod(path, 0)) + if os.Getuid() == 0 { + t.Skip("root reads a file of mode 0 itself; nothing to escalate") + } + forget, catted := "", "" + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "cat": + catted = strings.Join(args, " ") + return "daily=7\nweekly=4\nmonthly=24\n", nil + case name == "restic" && args[5] == "forget": + forget = "restic " + strings.Join(args[5:], " ") + case name == "restic" && args[5] == "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + } + return "", nil + } + b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet} + _, err := b.BackUp(context.Background(), "") + must(t, err) + if catted != "-- "+path { + t.Errorf("read %q as root, want %q", catted, "-- "+path) + } + if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; forget != want { + t.Errorf("ran %q, want %q", forget, want) } } diff --git a/modules/restic/go.mod b/modules/restic/go.mod index f010f2fc..7ec16d84 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -2,4 +2,4 @@ module restic go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c diff --git a/modules/restic/go.sum b/modules/restic/go.sum index b474419e..dfc73823 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/restic/module.json b/modules/restic/module.json index da5af259..57e93053 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -12,6 +12,11 @@ ] } ], + "tools": [ + "backup_health", + "backup_delete_retired", + "backup_deleted" + ], "own-secrets": { "repository": "${dir:state}/repository.secret" }, @@ -92,6 +97,53 @@ "id": "sqlite", "type": "package", "package": "sqlite" + }, + { + "id": "night", + "type": "process", + "name": "restic-night", + "artifact": "tools", + "run": [ + "./restic-backups", + "night" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "measure", + "type": "process", + "name": "restic-measure", + "artifact": "tools", + "run": [ + "./restic-backups", + "measure" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "unit" + } } ], "build": { From cbe9fa7a2c1924135e9d6d5d50e7d9a5fcc547ab Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 12:14:54 +0200 Subject: [PATCH 2/2] restic: raise a night that never happens, and have the mesh write the retention readable (novox/hq issue 454) A hung night, a lock held for hours or a lock that cannot be taken left no failed night behind, so it stayed healthy. The night's process now counts how long the machine was up since its newest good night, by the monotonic clock so a sleeping laptop is not blamed, and past 26 hours it is raised. The retention file is written 0644 and root's instead of read through sudo: the account reads it and still cannot change it. An unreadable file names its cause, and the healthy answer says only what was seen. --- modules/restic/cmd/restic-backups/awake.go | 102 ++++++++++++++++++ modules/restic/cmd/restic-backups/backups.go | 20 ++-- modules/restic/cmd/restic-backups/health.go | 78 ++++++++++++-- .../restic/cmd/restic-backups/health_test.go | 74 +++++++++++++ modules/restic/cmd/restic-backups/main.go | 1 + .../restic/cmd/restic-backups/retention.go | 56 +++------- .../cmd/restic-backups/retention_test.go | 61 +++++++---- modules/restic/module.json | 2 +- 8 files changed, 310 insertions(+), 84 deletions(-) create mode 100644 modules/restic/cmd/restic-backups/awake.go diff --git a/modules/restic/cmd/restic-backups/awake.go b/modules/restic/cmd/restic-backups/awake.go new file mode 100644 index 00000000..d11ede4c --- /dev/null +++ b/modules/restic/cmd/restic-backups/awake.go @@ -0,0 +1,102 @@ +package main + +// A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a +// restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed +// night behind, so the health also asks how long this machine has been up, with the night's process running, +// since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has +// missed nothing it could have done, and the night's process runs one soon after it starts again. +// +// The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while +// the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time +// the process was not running is not counted either: that is its unit's to say, judged by liveness. + +import ( + "encoding/json" + "os" + "path/filepath" + "time" +) + +// Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours +// a night may take or be late by. +const Overdue = 26 * time.Hour + +// LookEvery is how often the night's process counts; Stale how old its count may be before it is said. +const ( + LookEvery = time.Minute + Stale = 10 * time.Minute +) + +// Awake is the night's process's count: since which good night (zero before the first), how long the +// machine was up with the process running, when the process started and when it last counted. +type Awake struct { + From time.Time `json:"from"` + Up time.Duration `json:"up"` + Started time.Time `json:"started"` + Looked time.Time `json:"looked"` +} + +func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") } + +// LastAwake is the count as the night's process last wrote it; nil before it ever did. +func (b *Backups) LastAwake() *Awake { + raw, err := os.ReadFile(b.awakeFile()) + if err != nil { + return nil + } + var a Awake + if json.Unmarshal(raw, &a) != nil { + return nil + } + return &a +} + +func (b *Backups) keepAwake(a Awake) { + raw, _ := json.MarshalIndent(a, "", " ") + if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording how long this machine was up since its last good night failed: %v", err) + } +} + +// newestGood is the newest good night of any module; zero before the first. +func newestGood(nights map[string]Night) time.Time { + var newest time.Time + for _, n := range nights { + if n.OK && n.At.After(newest) { + newest = n.At + } + } + return newest +} + +// Count adds one look: up is how long the machine was awake since the last one. A newer good night starts +// the count again. Pure. +func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake { + if !good.Equal(a.From) { + a.From, a.Up = good, 0 + } + if up > 0 { + a.Up += up + } + a.Looked = now.UTC() + return a +} + +// counting is the night's process's count, once a minute while it runs. It carries on from what an earlier +// process left, so a restart neither forgives an overdue night nor starts one fresh. +func counting(b *Backups) { + a := Awake{} + if was := b.LastAwake(); was != nil { + a = *was + } + a.Started = time.Now().UTC() + last := time.Now() + for { + now := time.Now() + // Both readings carry the monotonic clock, which does not run while the machine is suspended. + a = a.Count(newestGood(b.Nights()), now.Sub(last), now) + last = now + b.keepAwake(a) + time.Sleep(LookEvery) + } +} diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index d6439e10..aba6086e 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -184,7 +184,6 @@ type Backups struct { Run Runner Now func() time.Time Say func(format string, args ...any) - read retentionRead } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a @@ -333,11 +332,6 @@ func snapshotOf(out string) string { // BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and // keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { - release, err := b.hold(nightLock) - if err != nil { - return nil, err - } - defer release() run := b.LastRun() run.At = b.Now().UTC() failed := func(err error) (map[string]Night, error) { @@ -345,6 +339,11 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er b.keepRun(run) return nil, err } + release, err := b.hold(nightLock) + if err != nil { + return failed(err) + } + defer release() if err := b.ensureRepository(ctx); err != nil { return failed(err) } @@ -701,11 +700,6 @@ func nextNight(now time.Time, hour int) time.Time { // missedANight is whether a night was missed: the newest good night of any module is older than a // day and a bit — the machine was off, or this module was not running, at the hour. func missedANight(nights map[string]Night, now time.Time) bool { - var newest time.Time - for _, n := range nights { - if n.OK && n.At.After(newest) { - newest = n.At - } - } - return newest.IsZero() || now.Sub(newest) > 26*time.Hour + newest := newestGood(nights) + return newest.IsZero() || now.Sub(newest) > Overdue } diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go index c5be2466..1fd71ec0 100644 --- a/modules/restic/cmd/restic-backups/health.go +++ b/modules/restic/cmd/restic-backups/health.go @@ -5,8 +5,9 @@ package main // only — a typo in a setting let the repository grow until a disk condition fired. // // Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run -// at all, when the last thinning failed, or when the retention file is missing, cannot be read or is not in -// the form the mesh writes. A retention setting the night refuses is the operator's to give again: the +// at all, when no night has backed anything up while this machine was up for more than a day (awake.go), when +// the night's process has stopped counting, when the last thinning failed, or when the retention file is +// missing, cannot be read or is not in the form the mesh writes. A retention setting the night refuses is the operator's to give again: the // answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator // condition. The node-engine asks this through the node tools for the night's process; the controller // raises module.restic..unhealthy on the second look that says so, with these words. @@ -37,6 +38,17 @@ type Seen struct { Good map[string]time.Time Run Run Retention InForce + // Awake is the night's process's count of how long this machine was up since its newest good night; nil + // before the process ever counted. + Awake *Awake + Now time.Time +} + +func hours(d time.Duration) string { + if d < 2*time.Hour { + return fmt.Sprintf("%d minutes", int(d.Minutes())) + } + return fmt.Sprintf("%d hours", int(d.Hours())) } func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") } @@ -53,6 +65,26 @@ func Judge(s Seen) Health { found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+ "names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed)) } + if a := s.Awake; a != nil { + newest := "no night has backed anything up here yet" + if !a.From.IsZero() { + newest = "the newest good night was at " + clock(a.From) + } + switch { + case s.Now.Sub(a.Looked) > Stale: + found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+ + "night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+ + "on this machine.", clock(a.Looked), hours(Stale), newest)) + case a.Up > Overdue: + found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+ + "night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+ + "the repository, or never starts leaves no failed night behind; read the journal of restic-night "+ + "(node-service-manager.journal), then take one with node-backup.now on this machine.", newest, hours(a.Up))) + default: + said = append(said, fmt.Sprintf("Up %s with the night's process running since the newest good night; "+ + "said when that passes %s.", hours(a.Up), hours(Overdue))) + } + } failed := map[string]Night{} for _, d := range s.Declared { if n, ok := s.Nights[d.Module]; ok && !n.OK { @@ -94,13 +126,23 @@ func Judge(s Seen) Health { said = append(said, line) waits = append(waits, health.Wait{Part: "night", Setting: r.Setting, What: "a count from 1 to 9999 for " + r.Setting}) + case strings.Contains(r.Refused, Unreadable): + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. The mesh writes the file readable (0644) and root's; one the account cannot "+ + "read was written by an older build of this module, or changed on the machine, and the next push of this "+ + "machine (mesh-controller.push) writes it readable again.", r.Refused)) case strings.HasSuffix(r.Refused, Missing): found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ "kept until it is there again, rather than thinned by a number nobody set; push this machine "+ "(mesh-controller.push) so the mesh writes it again.", r.Refused)) + case strings.Contains(r.Refused, RetentionFile+": "): + // Read, and not in the form the mesh writes: something other than the mesh wrote it. + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. The mesh writes only daily=, weekly= and monthly=, so something else wrote "+ + "this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) default: found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ - "kept, so the repository grows; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) + "kept, so the repository grows. Fix what the error names.", r.Refused)) } if len(found) > 0 { why := strings.Join(found, " ") @@ -112,22 +154,36 @@ func Judge(s Seen) Health { if len(waits) > 0 { return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said} } - why := "Every module declared here was backed up at its last night, and the last thinning succeeded." + return Health{Answer: health.Healthy(healthySaid(s, said)), Said: said} +} + +// healthySaid is what was seen, and only that, when nothing is wrong. +func healthySaid(s Seen, said []string) string { + var parts []string switch { case len(s.Declared) == 0: - why = "No module here declares anything to back up." - case len(s.Nights) == 0: - why = "No night has run here yet." + parts = append(parts, "No module here declares anything to back up.") + case newestGood(s.Nights).IsZero(): + parts = append(parts, "No good night is recorded here yet.") + default: + parts = append(parts, fmt.Sprintf("No module declared here failed its last night; the newest good night was at %s.", + clock(newestGood(s.Nights)))) } - if len(said) > 0 { - why += " " + said[0] + switch t := s.Run.Thinned; { + case t == nil: + parts = append(parts, "No thinning is recorded yet.") + case t.OK && t.Kept != nil: + parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points.", + clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly)) + case t.Refused != "": + parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At))) } - return Health{Answer: health.Healthy(why), Said: said} + return strings.Join(append(parts, said...), " ") } // Health reads what the nights left and the retention as it is now, and judges them. func (b *Backups) Health(ctx context.Context) Health { declared, err := b.Declared() return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), - Retention: b.inForce(ctx)}) + Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()}) } diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index f0c97115..452e4e4e 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -306,3 +306,77 @@ func slicesContains(s []string, v string) bool { } return false } + +// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a +// day since the newest good night, counted only while it was awake (novox/hq issue 454). +func TestANightThatNeverHappensIsUnhealthy(t *testing.T) { + good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, + Retention: InForce{Retention: &TodaysRetention}, Now: now} + + seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)} + h := Judge(seen) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") { + t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why) + } + + // A laptop asleep most of those two days: up 20 hours of them, nothing is overdue. + seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)} + if h := Judge(seen); h.State != health.StateHealthy { + t.Fatalf("up 20 hours: %s: %q", h.State, h.Why) + } + + // The process stopped counting: said, not taken for a night that is on time. + seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)} + if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { + t.Fatalf("a count an hour old: %s: %q", h.State, h.Why) + } +} + +func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) { + first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC) + a := Awake{}.Count(first, time.Minute, now) + a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake + if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) { + t.Fatalf("counted %+v", a) + } + a = a.Count(first.Add(24*time.Hour), time.Minute, now) + if a.Up != time.Minute { + t.Fatalf("a newer good night did not start the count again: %+v", a) + } +} + +// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one +// called a success. +func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) { + good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good} + h := Judge(seen) + if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") { + t.Fatalf("with no thinning recorded: %q", h.Why) + } + seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""} + if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") { + t.Fatalf("after a refused thinning: %q", h.Why) + } + seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24}} + if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") { + t.Fatalf("after a thinning: %q", h.Why) + } +} + +// The lock that cannot be taken is a night that did not run, and recorded as one. +func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) { + b, _ := nightAt(t) + must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write + if _, err := b.BackUp(context.Background(), ""); err == nil { + t.Fatal("a night ran without its lock") + } + if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") { + t.Fatalf("%s: %q", h.State, h.Why) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 648b9d49..2e9d4610 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -69,6 +69,7 @@ func nights(b *Backups) { if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { hour = h } + go counting(b) if missedANight(b.Nights(), time.Now()) { time.Sleep(10 * time.Minute) night(b) diff --git a/modules/restic/cmd/restic-backups/retention.go b/modules/restic/cmd/restic-backups/retention.go index 90da6b1c..26d23bf7 100644 --- a/modules/restic/cmd/restic-backups/retention.go +++ b/modules/restic/cmd/restic-backups/retention.go @@ -16,8 +16,6 @@ import ( "regexp" "strconv" "strings" - "sync" - "syscall" ) // Retention is how many restore points each period keeps. @@ -87,55 +85,31 @@ func (e *SettingRefused) Error() string { // rather than thinned by 14/8/6, which would thin by a number nobody set (novox/hq issue 454). const Missing = "is missing" -// retentionRead is the last read of the file, by what its stat said, so a health look every minute does not -// run sudo every minute while nothing changed. -type retentionRead struct { - mu sync.Mutex - key string - keep Retention - err error -} +// Unreadable is said of a retention file this program's account may not read. +const Unreadable = "cannot be read by the account this module runs as" // retention is this machine's, as the file the mesh wrote says; an error when it is missing, cannot be // read, or does not read, and a *SettingRefused within it when one setting is what is wrong. // -// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339) and this program runs as the -// operator account, so what the account may not read is read as root, as the declared directories are -// (novox/hq issue 454: until then every night refused it as unreadable and thinned nothing). -func (b *Backups) retention(ctx context.Context) (Retention, error) { +// The mesh writes the file root's and readable by everyone (0644): the account the night runs as reads it +// and cannot change it, and the counts are no secret (novox/hq issue 454 — written 0600 before, which the +// account could not read, so every night would have refused it and thinned nothing). +func (b *Backups) retention(context.Context) (Retention, error) { path := filepath.Join(b.Where.State, RetentionFile) - info, err := os.Stat(path) - if errors.Is(err, fs.ErrNotExist) { - return Retention{}, fmt.Errorf("%s %s", path, Missing) - } - if err != nil { - return Retention{}, err - } - key := fmt.Sprint(info.ModTime().UnixNano(), info.Size(), info.Mode()) - if st, ok := info.Sys().(*syscall.Stat_t); ok { - key += fmt.Sprint(" ", st.Ino, st.Ctim.Nano()) - } - b.read.mu.Lock() - defer b.read.mu.Unlock() - if b.read.key == key { - return b.read.keep, b.read.err - } raw, err := os.ReadFile(path) - if errors.Is(err, fs.ErrPermission) { - var out string - out, err = b.Run(ctx, "cat", "--", path) - raw = []byte(out) - } - if err != nil { - // Not kept: a read that failed may succeed at the next look. - return Retention{}, fmt.Errorf("%s cannot be read: %w", path, err) + switch { + case errors.Is(err, fs.ErrNotExist): + return Retention{}, fmt.Errorf("%s %s", path, Missing) + case errors.Is(err, fs.ErrPermission): + return Retention{}, fmt.Errorf("%s %s (%v)", path, Unreadable, errors.Unwrap(err)) + case err != nil: + return Retention{}, err } keep, err := ParseRetention(string(raw)) if err != nil { - err = fmt.Errorf("%s: %w", path, err) + return Retention{}, fmt.Errorf("%s: %w", path, err) } - b.read.key, b.read.keep, b.read.err = key, keep, err - return keep, err + return keep, nil } // InForce is the retention a night thins by, or why it thins nothing: what `backed-up` says beside each diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index 7449ba30..9b489b80 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -69,23 +69,18 @@ func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) { } } -// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339), and the night runs as the -// operator account: what the account cannot read is read as root, as every other root's file here is -// (novox/hq issue 454). Before, every machine's night refused it as unreadable and thinned nothing. -func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) { - where := placed(t, "# pg\npath /\n") - path := filepath.Join(where.State, RetentionFile) - must(t, os.WriteFile(path, []byte("daily=7\nweekly=4\nmonthly=24\n"), 0o600)) - must(t, os.Chmod(path, 0)) +// A file the account the night runs as may not read — the mode an older build of this module had the mesh +// write it in (0600, root's) — thins nothing, and says why (novox/hq issue 454). +func TestARetentionTheAccountCannotReadThinsNothingAndSaysWhy(t *testing.T) { if os.Getuid() == 0 { - t.Skip("root reads a file of mode 0 itself; nothing to escalate") + t.Skip("root reads a file of mode 0 itself") } - forget, catted := "", "" + where := placed(t, "# pg\npath /\n") + must(t, os.Chmod(filepath.Join(where.State, RetentionFile), 0)) + forget := "" + var said []string run := func(_ context.Context, name string, args ...string) (string, error) { switch { - case name == "cat": - catted = strings.Join(args, " ") - return "daily=7\nweekly=4\nmonthly=24\n", nil case name == "restic" && args[5] == "forget": forget = "restic " + strings.Join(args[5:], " ") case name == "restic" && args[5] == "backup": @@ -93,14 +88,44 @@ func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) { } return "", nil } - b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet} + b := &Backups{Where: where, Run: run, Now: time.Now, Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} _, err := b.BackUp(context.Background(), "") must(t, err) - if catted != "-- "+path { - t.Errorf("read %q as root, want %q", catted, "-- "+path) + if forget != "" { + t.Errorf("thinned by a file it could not read: %q", forget) } - if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; forget != want { - t.Errorf("ran %q, want %q", forget, want) + if all := strings.Join(said, "\n"); !strings.Contains(all, Unreadable) || !strings.Contains(all, "every restore point is kept") { + t.Errorf("said %q", all) + } + h := b.Health(context.Background()) + if h.State != "unhealthy" || !strings.Contains(h.Why, Unreadable) || !strings.Contains(h.Why, "0644") { + t.Errorf("%s: %q, want it unhealthy naming the cause", h.State, h.Why) + } +} + +// The mesh writes the file readable, and root's: the account reads it and cannot change it. +func TestTheManifestWritesTheRetentionReadable(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + must(t, err) + var m struct { + Resources []map[string]any `json:"resources"` + } + must(t, json.Unmarshal(raw, &m)) + file, night := -1, -1 + for i, r := range m.Resources { + switch r["id"] { + case "retention": + file = i + if r["mode"] != "0644" || r["owner"] != nil { + t.Errorf("the retention file is mode %v owner %v, want 0644 and root's", r["mode"], r["owner"]) + } + case "night": + night = i + } + } + // Applied in declared order: the rollout that brings the night's process makes the file readable first. + if file < 0 || night < 0 || file > night { + t.Errorf("the retention file is declared at %d and the night at %d", file, night) } } diff --git a/modules/restic/module.json b/modules/restic/module.json index 57e93053..544c0b81 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -84,7 +84,7 @@ "id": "retention", "type": "file", "path": "${dir:state}/retention.conf", - "mode": "0600", + "mode": "0644", "trusted": true, "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly and keep-monthly (novox/hq issue 416). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\n" },