From d649309895b0f8a3e4b6b7235d3d2f6feaed4045 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:56:23 +0200 Subject: [PATCH] restic: raise a failed backup, a failed thinning and a refused retention as the module's health (novox/hq issue 454) A failed night was a journal line only, so a typo in a retention setting let the repository grow until a disk filled. The night and the measuring now run as processes the node-engine judges: the night by backup_health, which is unhealthy for a module not backed up, a night that did not run, a failed thinning or a missing retention file, and waits naming the setting when one is refused; the measuring by its unit. A file lock keeps a night and the tool runtime's now and restore apart. The retention file is root's alone, so the account reads it through sudo: until now every night would have refused it as unreadable. Every machine has the file, so a missing one is refused instead of thinned by 14/8/6. backed-up says the retention in force and the last thinning per module. --- modules/restic/cmd/restic-backups/backups.go | 135 +++++++- .../restic/cmd/restic-backups/backups_test.go | 2 + modules/restic/cmd/restic-backups/data.go | 19 +- modules/restic/cmd/restic-backups/health.go | 133 ++++++++ .../restic/cmd/restic-backups/health_test.go | 308 ++++++++++++++++++ modules/restic/cmd/restic-backups/main.go | 45 ++- .../restic/cmd/restic-backups/retention.go | 108 +++++- .../cmd/restic-backups/retention_test.go | 55 +++- modules/restic/go.mod | 2 +- modules/restic/go.sum | 4 +- modules/restic/module.json | 52 +++ 11 files changed, 807 insertions(+), 56 deletions(-) create mode 100644 modules/restic/cmd/restic-backups/health.go create mode 100644 modules/restic/cmd/restic-backups/health_test.go diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 3b67e781..d6439e10 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -9,9 +9,9 @@ // // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and // restored on its own. Everything lands in one repository on the machine — deduplicated, so every -// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, -// 8 weekly and 6 monthly unless the machine's settings say otherwise (retention.go). Against -// mistakes, not disasters: nothing leaves the machine. +// night is a complete restore point and only what changed costs space — and is thinned by the machine's +// settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters: +// nothing leaves the machine. How the last night and thinning went is the module's health (health.go). // // Root's: the dumps read every store and the repository holds every module's data, and the runtime // launching this binary runs as the operator's account, so restic and the run lines go through sudo @@ -32,7 +32,7 @@ import ( "slices" "strconv" "strings" - "sync" + "syscall" "time" ) @@ -178,13 +178,44 @@ func whereFromEnv() (Where, error) { } // Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never -// share a dump. +// share a dump — across the night's process and the tool runtime alike (hold). type Backups struct { Where Where Run Runner Now func() time.Time Say func(format string, args ...any) - mu sync.Mutex + read retentionRead +} + +// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a +// retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer +// keeps the other out. +const ( + nightLock = "night.lock" + measureLock = "measure.lock" +) + +// hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open +// file's, so two holders in one process exclude each other as two processes do. +func (b *Backups) hold(name string) (func(), error) { + f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err) + } + for { + err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX) + if !errors.Is(err, syscall.EINTR) { + break + } + } + if err != nil { + f.Close() + return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + f.Close() + }, nil } // exists is whether a path is there, asked as root: a store's directory is often its own user's @@ -299,16 +330,27 @@ func snapshotOf(out string) string { return id } -// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome. +// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and +// keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { - b.mu.Lock() - defer b.mu.Unlock() - if err := b.ensureRepository(ctx); err != nil { + release, err := b.hold(nightLock) + if err != nil { return nil, err } + defer release() + run := b.LastRun() + run.At = b.Now().UTC() + failed := func(err error) (map[string]Night, error) { + run.Failed = err.Error() + b.keepRun(run) + return nil, err + } + if err := b.ensureRepository(ctx); err != nil { + return failed(err) + } all, err := b.Declared() if err != nil { - return nil, err + return failed(err) } chosen := all if only != "" { @@ -321,6 +363,7 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } } if len(chosen) == 0 { + // Asked of one module that is not here: nothing ran, and the last night stands as it was. return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names)) } } @@ -331,16 +374,59 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } // A retention that does not read thins nothing: keeping too much costs space, thinning by a value // nobody meant loses restore points (novox/hq issue 416). - keep, err := b.retention() + thinned := Thinning{At: b.Now().UTC()} + keep, err := b.retention(ctx) if err != nil { + thinned.Refused = err.Error() b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err) } else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", "--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil { + thinned.Kept, thinned.Error = &keep, err.Error() b.Say("thinning the restore points failed, and every one is kept: %v", err) + } else { + thinned.Kept, thinned.OK = &keep, true } + run.Failed, run.Thinned = "", &thinned + b.keepRun(run) return outcome, nil } +// Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its +// thinning. Each module's own outcome is its Night. +type Run struct { + At time.Time `json:"at"` + // Failed is why it did not run at all: the repository would not open, or what to back up does not read. + Failed string `json:"failed,omitempty"` + Thinned *Thinning `json:"thinned,omitempty"` +} + +// Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing. +type Thinning struct { + At time.Time `json:"at"` + OK bool `json:"ok"` + Kept *Retention `json:"kept,omitempty"` + Refused string `json:"refused,omitempty"` + Error string `json:"error,omitempty"` +} + +func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") } + +// LastRun is how the last night went as a whole; empty before the first. +func (b *Backups) LastRun() Run { + var r Run + if raw, err := os.ReadFile(b.runFile()); err == nil { + _ = json.Unmarshal(raw, &r) + } + return r +} + +func (b *Backups) keepRun(r Run) { + raw, _ := json.MarshalIndent(r, "", " ") + if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording how the night went failed: %v", err) + } +} + func orNothing(names []string) string { if len(names) == 0 { return "nothing" @@ -376,6 +462,11 @@ type ModuleBackups struct { LastNight *Night `json:"lastNight"` RestorePoints int `json:"restorePoints"` Newest *Snapshot `json:"newest,omitempty"` + // Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned + // how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its + // own, so the answer stays the list the controller reads. + Retention InForce `json:"retention"` + Thinned *Thinning `json:"thinned,omitempty"` // Data is every item the module declares, measured, with its newest good backup (novox/hq ADR // 0233): what the controller's self-check reads. Data []ItemReport `json:"data"` @@ -396,6 +487,8 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, good := b.Good() measured := b.Measurements() snaps, _ := b.Snapshots(ctx, module) + inForce := b.inForce(ctx) + thinned := b.LastRun().Thinned out := []ModuleBackups{} listed := map[string]bool{} for _, d := range declared { @@ -403,7 +496,7 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, continue } listed[d.Module] = true - m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, + m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned, Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])} if n, ok := nights[d.Module]; ok { m.LastNight = &n @@ -440,8 +533,11 @@ type Restored struct { // state as restored/// (besideIsUnsafe). A target that already exists is refused, never // overwritten. func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) { - b.mu.Lock() - defer b.mu.Unlock() + release, err := b.hold(nightLock) + if err != nil { + return nil, err + } + defer release() mine, err := b.Snapshots(ctx, module) if err != nil { return nil, err @@ -583,9 +679,12 @@ func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) { // Check is the weekly look at the repository's own integrity, with a sample of the data read back. func (b *Backups) Check(ctx context.Context) error { - b.mu.Lock() - defer b.mu.Unlock() - _, err := b.restic(ctx, "check", "--read-data-subset", "5%") + release, err := b.hold(nightLock) + if err != nil { + return err + } + defer release() + _, err = b.restic(ctx, "check", "--read-data-subset", "5%") return err } diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go index 77b0dc1f..bf6ee69f 100644 --- a/modules/restic/cmd/restic-backups/backups_test.go +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -25,6 +25,8 @@ func placed(t *testing.T, declared string) Where { dir := t.TempDir() must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600)) must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600)) + // The retention file every machine has, as the mesh writes it with the defaults (novox/hq issue 454). + must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600)) return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"), PasswordFile: filepath.Join(dir, "pw"), State: dir} } diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index f0d6dd50..a7a3d5fc 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -157,8 +157,6 @@ func (b *Backups) Measurements() map[string]map[string]Measured { return out } -var measuring sync.Mutex - // The ways an item is measured. const ( measureWalk = "walk" @@ -364,8 +362,12 @@ func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } // measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the // hourly round reads counters, and a walk runs at most once a day. func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { - measuring.Lock() - defer measuring.Unlock() + // The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454). + release, err := b.hold(measureLock) + if err != nil { + return err + } + defer release() items, err := b.Items() if err != nil { return err @@ -552,8 +554,13 @@ func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm } func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) { - b.mu.Lock() - defer b.mu.Unlock() + release, err := b.hold(nightLock) + if err != nil { + d.Running, d.Done, d.Error = false, true, err.Error() + b.keepDeletion(path, d) + return + } + defer release() finish := func(err error) { d.Running, d.Done = false, true if err != nil { diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go new file mode 100644 index 00000000..c5be2466 --- /dev/null +++ b/modules/restic/cmd/restic-backups/health.go @@ -0,0 +1,133 @@ +package main + +// The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be +// loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines +// only — a typo in a setting let the repository grow until a disk condition fired. +// +// Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run +// at all, when the last thinning failed, or when the retention file is missing, cannot be read or is not in +// the form the mesh writes. A retention setting the night refuses is the operator's to give again: the +// answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator +// condition. The node-engine asks this through the node tools for the night's process; the controller +// raises module.restic..unhealthy on the second look that says so, with these words. + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why, +// waits), with what was found and what was said beside it. +type Health struct { + health.Answer + Found []string `json:"found,omitempty"` + Said []string `json:"said,omitempty"` +} + +// Seen is what the health is judged from. +type Seen struct { + Declared []Declared + DeclaredErr error + Nights map[string]Night + Good map[string]time.Time + Run Run + Retention InForce +} + +func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") } + +// Judge is the health of what was seen. Pure. +func Judge(s Seen) Health { + var found, said []string + var waits []health.Wait + if s.DeclaredErr != nil { + found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+ + "back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr)) + } + if s.Run.Failed != "" { + found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+ + "names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed)) + } + failed := map[string]Night{} + for _, d := range s.Declared { + if n, ok := s.Nights[d.Module]; ok && !n.OK { + failed[d.Module] = n + } + } + names := make([]string, 0, len(failed)) + for m := range failed { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + n := failed[m] + since := "it has no restore point here" + if g, ok := s.Good[m]; ok && !g.IsZero() { + since = "its newest restore point is of " + clock(g) + } + found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+ + `with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m)) + } + if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" { + found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. Nothing was forgotten and every "+ + "restore point is kept, so the repository grows until a thinning succeeds; fix what the error names, then "+ + "node-backup.now on this machine thins again.", clock(t.At), strings.TrimSuffix(t.Error, "."))) + } + r := s.Retention + switch { + case r.Refused == "" && r.Retention != nil: + said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module.", + r.Daily, r.Weekly, r.Monthly)) + case r.Setting != "": + value := "not set" + if r.Value != "" { + value = fmt.Sprintf("%q", r.Value) + } + line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+ + "nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+ + "(mesh-controller.settings).", r.Setting, value, r.Setting) + said = append(said, line) + waits = append(waits, health.Wait{Part: "night", Setting: r.Setting, + What: "a count from 1 to 9999 for " + r.Setting}) + case strings.HasSuffix(r.Refused, Missing): + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept until it is there again, rather than thinned by a number nobody set; push this machine "+ + "(mesh-controller.push) so the mesh writes it again.", r.Refused)) + default: + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) + } + if len(found) > 0 { + why := strings.Join(found, " ") + if len(waits) > 0 { + why += " " + said[len(said)-1] + } + return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said} + } + if len(waits) > 0 { + return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said} + } + why := "Every module declared here was backed up at its last night, and the last thinning succeeded." + switch { + case len(s.Declared) == 0: + why = "No module here declares anything to back up." + case len(s.Nights) == 0: + why = "No night has run here yet." + } + if len(said) > 0 { + why += " " + said[0] + } + return Health{Answer: health.Healthy(why), Said: said} +} + +// Health reads what the nights left and the retention as it is now, and judges them. +func (b *Backups) Health(ctx context.Context) Health { + declared, err := b.Declared() + return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), + Retention: b.inForce(ctx)}) +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go new file mode 100644 index 00000000..f0c97115 --- /dev/null +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -0,0 +1,308 @@ +package main + +// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and +// a refused retention each raise a condition naming the module and what to do, instead of a journal line +// nobody reads until a disk fills. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails +// with thinning when it is not empty. +type nightly struct { + failing map[string]bool + thinning string +} + +func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]: + return "", errors.New("the dump failed: no space left on device") + case name == "restic" && args[5] == "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + case name == "restic" && args[5] == "forget" && n.thinning != "": + return "", errors.New(n.thinning) + } + return "", nil +} + +func nightAt(t *testing.T) (*Backups, *nightly) { + t.Helper() + n := &nightly{failing: map[string]bool{}} + b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run, + Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet} + return b, n +} + +func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) { + b, n := nightAt(t) + n.failing["mail"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a failed backup is %s: %s", h.State, h.Why) + } + for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } + if strings.Contains(h.Why, "pg was NOT") { + t.Errorf("pg was backed up and is said as failed: %q", h.Why) + } + + // Backed up again, it clears. + n.failing["mail"] = false + _, err = b.BackUp(context.Background(), "mail") + must(t, err) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("backed up again, it is %s: %s", h.State, h.Why) + } +} + +func TestANightThatDidNotRunIsUnhealthy(t *testing.T) { + b, _ := nightAt(t) + b.Run = func(_ context.Context, name string, args ...string) (string, error) { + return "", errors.New("Fatal: wrong password or no key found") + } + if _, err := b.BackUp(context.Background(), ""); err == nil { + t.Fatal("a repository that will not open ran a night") + } + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") { + t.Fatalf("a night that did not run: %s: %q", h.State, h.Why) + } +} + +func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) { + b, n := nightAt(t) + n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively" + _, err := b.BackUp(context.Background(), "") + must(t, err) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a failed thinning is %s: %s", h.State, h.Why) + } + for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } + n.thinning = "" + _, err = b.BackUp(context.Background(), "") + must(t, err) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("thinned again, it is %s: %s", h.State, h.Why) + } +} + +// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283), +// and the controller says it as the module's needs-operator condition with the act. +func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) { + b, _ := nightAt(t) + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600)) + h := b.Health(context.Background()) + if h.State != health.StateWaits { + t.Fatalf("a refused setting is %s: %s", h.State, h.Why) + } + if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil { + t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits) + } + for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } +} + +// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the +// operator gave: it is a fault, and says to push the machine. +func TestAMissingRetentionFileIsUnhealthy(t *testing.T) { + b, _ := nightAt(t) + must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile))) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy { + t.Fatalf("a missing retention file is %s: %s", h.State, h.Why) + } + for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} { + if !strings.Contains(h.Why, want) { + t.Errorf("said %q, want it to hold %q", h.Why, want) + } + } +} + +// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for. +func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) { + b, n := nightAt(t) + n.failing["pg"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600)) + h := b.Health(context.Background()) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") { + t.Fatalf("%s: %q", h.State, h.Why) + } +} + +func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) { + b, _ := nightAt(t) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("before any night: %s: %s", h.State, h.Why) + } +} + +// A module no longer declared here is not held against the machine for a night it failed before. +func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) { + b, n := nightAt(t) + n.failing["mail"] = true + _, err := b.BackUp(context.Background(), "") + must(t, err) + must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600)) + if h := b.Health(context.Background()); h.State != health.StateHealthy { + t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why) + } +} + +// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning. +func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) { + b, n := nightAt(t) + n.thinning = "Fatal: repository is already locked" + _, err := b.BackUp(context.Background(), "") + must(t, err) + listed, err := b.BackedUp(context.Background(), "pg") + must(t, err) + if len(listed) != 1 { + t.Fatalf("listed %+v", listed) + } + raw, _ := json.Marshal(listed[0]) + var got struct { + Retention map[string]any `json:"retention"` + Thinned map[string]any `json:"thinned"` + } + must(t, json.Unmarshal(raw, &got)) + if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil { + t.Errorf("retention %v, want 14/8/6 not refused", got.Retention) + } + if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") { + t.Errorf("thinned %v, want the failure", got.Thinned) + } + + must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600)) + listed, err = b.BackedUp(context.Background(), "pg") + must(t, err) + raw, _ = json.Marshal(listed[0]) + got.Retention = nil + must(t, json.Unmarshal(raw, &got)) + if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" { + t.Errorf("retention %v, want it refused naming keep-daily", got.Retention) + } +} + +// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at +// a time across both, never two nights or a night and a restore over one repository. +func TestANightWaitsForAnotherProcesssNight(t *testing.T) { + b, _ := nightAt(t) + release, err := b.hold(nightLock) + must(t, err) + done := make(chan struct{}) + go func() { + other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet} + _, _ = other.BackUp(context.Background(), "") + close(done) + }() + select { + case <-done: + t.Fatal("a night ran while another held the repository") + case <-time.After(200 * time.Millisecond): + } + release() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the night did not run once the other let go") + } +} + +// The manifest runs the night and the measuring as processes of their own, as the operator account like the +// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as +// before), each given where everything is; the night judged by backup_health, the measuring by its unit — +// a tool check only beside a check of another kind (ADR 0240 rule 2). +func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + must(t, err) + var m struct { + Tools []string `json:"tools"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []struct { + Name string `json:"name"` + Env map[string]string `json:"env"` + } `json:"artifacts"` + } `json:"build"` + } + must(t, json.Unmarshal(raw, &m)) + served := map[string]bool{} + for _, tool := range tools(&Backups{}) { + served[tool.Name] = true + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which the bundle does not serve", tool) + } + } + env := m.Build.Artifacts[0].Env + want := map[string]struct{ run, kind, tool string }{ + "night": {"night", "tool", "backup_health"}, + "measure": {"measure", "unit", ""}, + } + for _, r := range m.Resources { + w, ok := want[r["id"].(string)] + if !ok || r["type"] != "process" { + continue + } + delete(want, r["id"].(string)) + run, _ := json.Marshal(r["run"]) + if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name { + t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"]) + } + if r["user"] != "${machine:account}" { + t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"]) + } + penv, _ := r["env"].(map[string]any) + for k, v := range env { + if penv[k] != v { + t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v) + } + } + h, _ := r["health"].(map[string]any) + if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) { + t.Errorf("%s is judged by %v", r["id"], h) + } + if w.tool != "" && !slicesContains(m.Tools, w.tool) { + t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool) + } + } + for id := range want { + t.Errorf("no process %s", id) + } +} + +func slicesContains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 508a69b4..648b9d49 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -1,7 +1,16 @@ -// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the -// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It -// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the -// live data — and, beside them, runs the night that happens without anyone asking. +// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, three ways to run: +// +// - with no argument, launched by the machine's tool runtime and speaking MCP to it over stdio through the +// Go SDK (ADR 0193, ADR 0198): the node-backup seat's three verbs — what is backed up, take one now, +// restore beside the live data — and the health check, backup_health; +// - `night`, a process the node-engine runs: the night that happens without anyone asking, judged by +// backup_health from what it leaves; +// - `measure`, a process the node-engine runs: the hourly measuring of the data declared here (ADR 0233), +// judged by its unit. +// +// The night and the measuring were goroutines of the tool runtime's process until novox/hq issue 454: a module +// raises a condition only through the health of something it runs (ADR 0240, ADR 0256), and a check by the +// module's own tool only beside a check of another kind, so each runs as a process of its own. // // stdout is the MCP channel; everything this module says, it says on stderr. package main @@ -31,11 +40,24 @@ func main() { os.Exit(1) } b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say} - go nights(b) - go measurements(b) - if err := stdio.Serve("", tools(b)); err != nil { - say("%v", err) - os.Exit(1) + way := "" + if len(os.Args) > 1 { + way = os.Args[1] + } + switch way { + case "night": + nights(b) + case "measure": + measurements(b) + case "": + if err := stdio.Serve("", tools(b)); err != nil { + say("%v", err) + os.Exit(1) + } + default: + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+ + "measure it measures the data declared here", way) + os.Exit(2) } } @@ -154,6 +176,11 @@ func tools(b *Backups) []stdio.Tool { } return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path")) }), + {Name: "backup_health", Description: "Whether this machine's backups hold, as the module's health check " + + "(novox/hq ADR 0240, issue 454): unhealthy when a module was NOT backed up at its last night, the last night " + + "did not run, the last thinning failed, or the retention file is missing or unreadable — each said with what " + + "to do; waits, naming the setting, when a retention setting is refused. (r)", + Run: func(map[string]any) (any, error) { return b.Health(context.Background()), nil }}, // Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup // delete` asks these. Not seat verbs — nobody else calls them. {Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " + diff --git a/modules/restic/cmd/restic-backups/retention.go b/modules/restic/cmd/restic-backups/retention.go index 8c37ee0d..90da6b1c 100644 --- a/modules/restic/cmd/restic-backups/retention.go +++ b/modules/restic/cmd/restic-backups/retention.go @@ -7,20 +7,28 @@ package main // refused here and never handed to restic (issue 346: a raw setting is an injection point). import ( + "context" "errors" "fmt" + "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" + "sync" + "syscall" ) // Retention is how many restore points each period keeps. -type Retention struct{ Daily, Weekly, Monthly int } +type Retention struct { + Daily int `json:"daily"` + Weekly int `json:"weekly"` + Monthly int `json:"monthly"` +} // TodaysRetention is the rotation before it was a preference (novox/hq ADR 0214), and still the -// manifest's default. +// manifest's default — never a fallback for a file that is not there (novox/hq issue 454). var TodaysRetention = Retention{14, 8, 6} // RetentionFile is the file the mesh writes the retention into, in this module's state. @@ -52,32 +60,104 @@ func ParseRetention(text string) (Retention, error) { case seen[k] != 0: return Retention{}, fmt.Errorf("%s is given twice", k) case !count.MatchString(v): - return Retention{}, fmt.Errorf("the %s setting is %q; it is a count from 1 to 9999", setting, v) + return Retention{}, &SettingRefused{Setting: setting, Value: v} } seen[k], _ = strconv.Atoi(v) } for _, k := range []string{"daily", "weekly", "monthly"} { if seen[k] == 0 { - return Retention{}, fmt.Errorf("the %s setting is missing", settingOf[k]) + return Retention{}, &SettingRefused{Setting: settingOf[k]} } } return Retention{seen["daily"], seen["weekly"], seen["monthly"]}, nil } -// retention is this machine's: today's where the module was placed by a definition older than the -// file, else what the file says. -func (b *Backups) retention() (Retention, error) { +// SettingRefused is one retention setting the night refuses: the operator's to give again. +type SettingRefused struct{ Setting, Value string } + +func (e *SettingRefused) Error() string { + if e.Value == "" { + return fmt.Sprintf("the %s setting is missing", e.Setting) + } + return fmt.Sprintf("the %s setting is %q; it is a count from 1 to 9999", e.Setting, e.Value) +} + +// Missing is said of a retention file that is not there. Every machine has had the file since mesh-catalog +// #200 rolled out (seen in each machine's plan, 2026-10-11), so a missing one is lost, not old: it is refused +// rather than thinned by 14/8/6, which would thin by a number nobody set (novox/hq issue 454). +const Missing = "is missing" + +// retentionRead is the last read of the file, by what its stat said, so a health look every minute does not +// run sudo every minute while nothing changed. +type retentionRead struct { + mu sync.Mutex + key string + keep Retention + err error +} + +// retention is this machine's, as the file the mesh wrote says; an error when it is missing, cannot be +// read, or does not read, and a *SettingRefused within it when one setting is what is wrong. +// +// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339) and this program runs as the +// operator account, so what the account may not read is read as root, as the declared directories are +// (novox/hq issue 454: until then every night refused it as unreadable and thinned nothing). +func (b *Backups) retention(ctx context.Context) (Retention, error) { path := filepath.Join(b.Where.State, RetentionFile) - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return TodaysRetention, nil + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return Retention{}, fmt.Errorf("%s %s", path, Missing) } if err != nil { return Retention{}, err } - r, err := ParseRetention(string(raw)) - if err != nil { - return Retention{}, fmt.Errorf("%s: %w", path, err) + key := fmt.Sprint(info.ModTime().UnixNano(), info.Size(), info.Mode()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + key += fmt.Sprint(" ", st.Ino, st.Ctim.Nano()) } - return r, nil + b.read.mu.Lock() + defer b.read.mu.Unlock() + if b.read.key == key { + return b.read.keep, b.read.err + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrPermission) { + var out string + out, err = b.Run(ctx, "cat", "--", path) + raw = []byte(out) + } + if err != nil { + // Not kept: a read that failed may succeed at the next look. + return Retention{}, fmt.Errorf("%s cannot be read: %w", path, err) + } + keep, err := ParseRetention(string(raw)) + if err != nil { + err = fmt.Errorf("%s: %w", path, err) + } + b.read.key, b.read.keep, b.read.err = key, keep, err + return keep, err +} + +// InForce is the retention a night thins by, or why it thins nothing: what `backed-up` says beside each +// module and what the health judges. +type InForce struct { + *Retention + Refused string `json:"refused,omitempty"` + // Setting is the setting a refusal names, when one setting is what is wrong. + Setting string `json:"setting,omitempty"` + // Value is what that setting is now. + Value string `json:"-"` +} + +func (b *Backups) inForce(ctx context.Context) InForce { + keep, err := b.retention(ctx) + if err == nil { + return InForce{Retention: &keep} + } + f := InForce{Refused: err.Error()} + var s *SettingRefused + if errors.As(err, &s) { + f.Setting, f.Value = s.Setting, s.Value + } + return f } diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index 4969d690..7449ba30 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -21,8 +21,11 @@ import ( func thinning(t *testing.T, retention *string) (string, []string) { t.Helper() where := placed(t, "# pg\npath /\n") + path := filepath.Join(where.State, RetentionFile) if retention != nil { - must(t, os.WriteFile(filepath.Join(where.State, "retention.conf"), []byte(*retention), 0o600)) + must(t, os.WriteFile(path, []byte(*retention), 0o600)) + } else { + must(t, os.Remove(path)) } forget := "" run := func(_ context.Context, name string, args ...string) (string, error) { @@ -53,11 +56,51 @@ func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) { } } -// A machine placed by a definition older than the file keeps today's rotation. -func TestWithoutTheFileANightKeepsTodaysRetention(t *testing.T) { - got, _ := thinning(t, nil) - if want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6"; got != want { - t.Errorf("ran %q, want %q", got, want) +// Every machine has had the file since mesh-catalog #200 rolled out (novox/hq issue 454), so a missing one is +// a fault, never a reason to thin by 14/8/6: a lost file would otherwise thin by a number nobody set. +func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) { + got, said := thinning(t, nil) + if got != "" { + t.Errorf("ran %q without a retention file", got) + } + all := strings.Join(said, "\n") + if !strings.Contains(all, "retention.conf is missing") || !strings.Contains(all, "every restore point is kept") { + t.Errorf("said %q, want it to say the file is missing and every restore point is kept", all) + } +} + +// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339), and the night runs as the +// operator account: what the account cannot read is read as root, as every other root's file here is +// (novox/hq issue 454). Before, every machine's night refused it as unreadable and thinned nothing. +func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) { + where := placed(t, "# pg\npath /\n") + path := filepath.Join(where.State, RetentionFile) + must(t, os.WriteFile(path, []byte("daily=7\nweekly=4\nmonthly=24\n"), 0o600)) + must(t, os.Chmod(path, 0)) + if os.Getuid() == 0 { + t.Skip("root reads a file of mode 0 itself; nothing to escalate") + } + forget, catted := "", "" + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "cat": + catted = strings.Join(args, " ") + return "daily=7\nweekly=4\nmonthly=24\n", nil + case name == "restic" && args[5] == "forget": + forget = "restic " + strings.Join(args[5:], " ") + case name == "restic" && args[5] == "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + } + return "", nil + } + b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet} + _, err := b.BackUp(context.Background(), "") + must(t, err) + if catted != "-- "+path { + t.Errorf("read %q as root, want %q", catted, "-- "+path) + } + if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; forget != want { + t.Errorf("ran %q, want %q", forget, want) } } diff --git a/modules/restic/go.mod b/modules/restic/go.mod index f010f2fc..7ec16d84 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -2,4 +2,4 @@ module restic go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c diff --git a/modules/restic/go.sum b/modules/restic/go.sum index b474419e..dfc73823 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/restic/module.json b/modules/restic/module.json index da5af259..57e93053 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -12,6 +12,11 @@ ] } ], + "tools": [ + "backup_health", + "backup_delete_retired", + "backup_deleted" + ], "own-secrets": { "repository": "${dir:state}/repository.secret" }, @@ -92,6 +97,53 @@ "id": "sqlite", "type": "package", "package": "sqlite" + }, + { + "id": "night", + "type": "process", + "name": "restic-night", + "artifact": "tools", + "run": [ + "./restic-backups", + "night" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "measure", + "type": "process", + "name": "restic-measure", + "artifact": "tools", + "run": [ + "./restic-backups", + "measure" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "unit" + } } ], "build": {