diff --git a/modules/restic/cmd/restic-backups/awake.go b/modules/restic/cmd/restic-backups/awake.go new file mode 100644 index 00000000..d11ede4c --- /dev/null +++ b/modules/restic/cmd/restic-backups/awake.go @@ -0,0 +1,102 @@ +package main + +// A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a +// restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed +// night behind, so the health also asks how long this machine has been up, with the night's process running, +// since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has +// missed nothing it could have done, and the night's process runs one soon after it starts again. +// +// The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while +// the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time +// the process was not running is not counted either: that is its unit's to say, judged by liveness. + +import ( + "encoding/json" + "os" + "path/filepath" + "time" +) + +// Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours +// a night may take or be late by. +const Overdue = 26 * time.Hour + +// LookEvery is how often the night's process counts; Stale how old its count may be before it is said. +const ( + LookEvery = time.Minute + Stale = 10 * time.Minute +) + +// Awake is the night's process's count: since which good night (zero before the first), how long the +// machine was up with the process running, when the process started and when it last counted. +type Awake struct { + From time.Time `json:"from"` + Up time.Duration `json:"up"` + Started time.Time `json:"started"` + Looked time.Time `json:"looked"` +} + +func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") } + +// LastAwake is the count as the night's process last wrote it; nil before it ever did. +func (b *Backups) LastAwake() *Awake { + raw, err := os.ReadFile(b.awakeFile()) + if err != nil { + return nil + } + var a Awake + if json.Unmarshal(raw, &a) != nil { + return nil + } + return &a +} + +func (b *Backups) keepAwake(a Awake) { + raw, _ := json.MarshalIndent(a, "", " ") + if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording how long this machine was up since its last good night failed: %v", err) + } +} + +// newestGood is the newest good night of any module; zero before the first. +func newestGood(nights map[string]Night) time.Time { + var newest time.Time + for _, n := range nights { + if n.OK && n.At.After(newest) { + newest = n.At + } + } + return newest +} + +// Count adds one look: up is how long the machine was awake since the last one. A newer good night starts +// the count again. Pure. +func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake { + if !good.Equal(a.From) { + a.From, a.Up = good, 0 + } + if up > 0 { + a.Up += up + } + a.Looked = now.UTC() + return a +} + +// counting is the night's process's count, once a minute while it runs. It carries on from what an earlier +// process left, so a restart neither forgives an overdue night nor starts one fresh. +func counting(b *Backups) { + a := Awake{} + if was := b.LastAwake(); was != nil { + a = *was + } + a.Started = time.Now().UTC() + last := time.Now() + for { + now := time.Now() + // Both readings carry the monotonic clock, which does not run while the machine is suspended. + a = a.Count(newestGood(b.Nights()), now.Sub(last), now) + last = now + b.keepAwake(a) + time.Sleep(LookEvery) + } +} diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index d6439e10..aba6086e 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -184,7 +184,6 @@ type Backups struct { Run Runner Now func() time.Time Say func(format string, args ...any) - read retentionRead } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a @@ -333,11 +332,6 @@ func snapshotOf(out string) string { // BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and // keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { - release, err := b.hold(nightLock) - if err != nil { - return nil, err - } - defer release() run := b.LastRun() run.At = b.Now().UTC() failed := func(err error) (map[string]Night, error) { @@ -345,6 +339,11 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er b.keepRun(run) return nil, err } + release, err := b.hold(nightLock) + if err != nil { + return failed(err) + } + defer release() if err := b.ensureRepository(ctx); err != nil { return failed(err) } @@ -701,11 +700,6 @@ func nextNight(now time.Time, hour int) time.Time { // missedANight is whether a night was missed: the newest good night of any module is older than a // day and a bit — the machine was off, or this module was not running, at the hour. func missedANight(nights map[string]Night, now time.Time) bool { - var newest time.Time - for _, n := range nights { - if n.OK && n.At.After(newest) { - newest = n.At - } - } - return newest.IsZero() || now.Sub(newest) > 26*time.Hour + newest := newestGood(nights) + return newest.IsZero() || now.Sub(newest) > Overdue } diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go index c5be2466..1fd71ec0 100644 --- a/modules/restic/cmd/restic-backups/health.go +++ b/modules/restic/cmd/restic-backups/health.go @@ -5,8 +5,9 @@ package main // only — a typo in a setting let the repository grow until a disk condition fired. // // Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run -// at all, when the last thinning failed, or when the retention file is missing, cannot be read or is not in -// the form the mesh writes. A retention setting the night refuses is the operator's to give again: the +// at all, when no night has backed anything up while this machine was up for more than a day (awake.go), when +// the night's process has stopped counting, when the last thinning failed, or when the retention file is +// missing, cannot be read or is not in the form the mesh writes. A retention setting the night refuses is the operator's to give again: the // answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator // condition. The node-engine asks this through the node tools for the night's process; the controller // raises module.restic..unhealthy on the second look that says so, with these words. @@ -37,6 +38,17 @@ type Seen struct { Good map[string]time.Time Run Run Retention InForce + // Awake is the night's process's count of how long this machine was up since its newest good night; nil + // before the process ever counted. + Awake *Awake + Now time.Time +} + +func hours(d time.Duration) string { + if d < 2*time.Hour { + return fmt.Sprintf("%d minutes", int(d.Minutes())) + } + return fmt.Sprintf("%d hours", int(d.Hours())) } func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") } @@ -53,6 +65,26 @@ func Judge(s Seen) Health { found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+ "names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed)) } + if a := s.Awake; a != nil { + newest := "no night has backed anything up here yet" + if !a.From.IsZero() { + newest = "the newest good night was at " + clock(a.From) + } + switch { + case s.Now.Sub(a.Looked) > Stale: + found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+ + "night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+ + "on this machine.", clock(a.Looked), hours(Stale), newest)) + case a.Up > Overdue: + found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+ + "night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+ + "the repository, or never starts leaves no failed night behind; read the journal of restic-night "+ + "(node-service-manager.journal), then take one with node-backup.now on this machine.", newest, hours(a.Up))) + default: + said = append(said, fmt.Sprintf("Up %s with the night's process running since the newest good night; "+ + "said when that passes %s.", hours(a.Up), hours(Overdue))) + } + } failed := map[string]Night{} for _, d := range s.Declared { if n, ok := s.Nights[d.Module]; ok && !n.OK { @@ -94,13 +126,23 @@ func Judge(s Seen) Health { said = append(said, line) waits = append(waits, health.Wait{Part: "night", Setting: r.Setting, What: "a count from 1 to 9999 for " + r.Setting}) + case strings.Contains(r.Refused, Unreadable): + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. The mesh writes the file readable (0644) and root's; one the account cannot "+ + "read was written by an older build of this module, or changed on the machine, and the next push of this "+ + "machine (mesh-controller.push) writes it readable again.", r.Refused)) case strings.HasSuffix(r.Refused, Missing): found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ "kept until it is there again, rather than thinned by a number nobody set; push this machine "+ "(mesh-controller.push) so the mesh writes it again.", r.Refused)) + case strings.Contains(r.Refused, RetentionFile+": "): + // Read, and not in the form the mesh writes: something other than the mesh wrote it. + found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ + "kept, so the repository grows. The mesh writes only daily=, weekly= and monthly=, so something else wrote "+ + "this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) default: found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ - "kept, so the repository grows; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) + "kept, so the repository grows. Fix what the error names.", r.Refused)) } if len(found) > 0 { why := strings.Join(found, " ") @@ -112,22 +154,36 @@ func Judge(s Seen) Health { if len(waits) > 0 { return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said} } - why := "Every module declared here was backed up at its last night, and the last thinning succeeded." + return Health{Answer: health.Healthy(healthySaid(s, said)), Said: said} +} + +// healthySaid is what was seen, and only that, when nothing is wrong. +func healthySaid(s Seen, said []string) string { + var parts []string switch { case len(s.Declared) == 0: - why = "No module here declares anything to back up." - case len(s.Nights) == 0: - why = "No night has run here yet." + parts = append(parts, "No module here declares anything to back up.") + case newestGood(s.Nights).IsZero(): + parts = append(parts, "No good night is recorded here yet.") + default: + parts = append(parts, fmt.Sprintf("No module declared here failed its last night; the newest good night was at %s.", + clock(newestGood(s.Nights)))) } - if len(said) > 0 { - why += " " + said[0] + switch t := s.Run.Thinned; { + case t == nil: + parts = append(parts, "No thinning is recorded yet.") + case t.OK && t.Kept != nil: + parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points.", + clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly)) + case t.Refused != "": + parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At))) } - return Health{Answer: health.Healthy(why), Said: said} + return strings.Join(append(parts, said...), " ") } // Health reads what the nights left and the retention as it is now, and judges them. func (b *Backups) Health(ctx context.Context) Health { declared, err := b.Declared() return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), - Retention: b.inForce(ctx)}) + Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()}) } diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index f0c97115..452e4e4e 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -306,3 +306,77 @@ func slicesContains(s []string, v string) bool { } return false } + +// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a +// day since the newest good night, counted only while it was awake (novox/hq issue 454). +func TestANightThatNeverHappensIsUnhealthy(t *testing.T) { + good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, + Retention: InForce{Retention: &TodaysRetention}, Now: now} + + seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)} + h := Judge(seen) + if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") { + t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why) + } + + // A laptop asleep most of those two days: up 20 hours of them, nothing is overdue. + seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)} + if h := Judge(seen); h.State != health.StateHealthy { + t.Fatalf("up 20 hours: %s: %q", h.State, h.Why) + } + + // The process stopped counting: said, not taken for a night that is on time. + seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)} + if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { + t.Fatalf("a count an hour old: %s: %q", h.State, h.Why) + } +} + +func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) { + first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) + now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC) + a := Awake{}.Count(first, time.Minute, now) + a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake + if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) { + t.Fatalf("counted %+v", a) + } + a = a.Count(first.Add(24*time.Hour), time.Minute, now) + if a.Up != time.Minute { + t.Fatalf("a newer good night did not start the count again: %+v", a) + } +} + +// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one +// called a success. +func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) { + good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) + seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, + Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good} + h := Judge(seen) + if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") { + t.Fatalf("with no thinning recorded: %q", h.Why) + } + seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""} + if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") { + t.Fatalf("after a refused thinning: %q", h.Why) + } + seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24}} + if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") { + t.Fatalf("after a thinning: %q", h.Why) + } +} + +// The lock that cannot be taken is a night that did not run, and recorded as one. +func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) { + b, _ := nightAt(t) + must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write + if _, err := b.BackUp(context.Background(), ""); err == nil { + t.Fatal("a night ran without its lock") + } + if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") { + t.Fatalf("%s: %q", h.State, h.Why) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 648b9d49..2e9d4610 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -69,6 +69,7 @@ func nights(b *Backups) { if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { hour = h } + go counting(b) if missedANight(b.Nights(), time.Now()) { time.Sleep(10 * time.Minute) night(b) diff --git a/modules/restic/cmd/restic-backups/retention.go b/modules/restic/cmd/restic-backups/retention.go index 90da6b1c..26d23bf7 100644 --- a/modules/restic/cmd/restic-backups/retention.go +++ b/modules/restic/cmd/restic-backups/retention.go @@ -16,8 +16,6 @@ import ( "regexp" "strconv" "strings" - "sync" - "syscall" ) // Retention is how many restore points each period keeps. @@ -87,55 +85,31 @@ func (e *SettingRefused) Error() string { // rather than thinned by 14/8/6, which would thin by a number nobody set (novox/hq issue 454). const Missing = "is missing" -// retentionRead is the last read of the file, by what its stat said, so a health look every minute does not -// run sudo every minute while nothing changed. -type retentionRead struct { - mu sync.Mutex - key string - keep Retention - err error -} +// Unreadable is said of a retention file this program's account may not read. +const Unreadable = "cannot be read by the account this module runs as" // retention is this machine's, as the file the mesh wrote says; an error when it is missing, cannot be // read, or does not read, and a *SettingRefused within it when one setting is what is wrong. // -// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339) and this program runs as the -// operator account, so what the account may not read is read as root, as the declared directories are -// (novox/hq issue 454: until then every night refused it as unreadable and thinned nothing). -func (b *Backups) retention(ctx context.Context) (Retention, error) { +// The mesh writes the file root's and readable by everyone (0644): the account the night runs as reads it +// and cannot change it, and the counts are no secret (novox/hq issue 454 — written 0600 before, which the +// account could not read, so every night would have refused it and thinned nothing). +func (b *Backups) retention(context.Context) (Retention, error) { path := filepath.Join(b.Where.State, RetentionFile) - info, err := os.Stat(path) - if errors.Is(err, fs.ErrNotExist) { - return Retention{}, fmt.Errorf("%s %s", path, Missing) - } - if err != nil { - return Retention{}, err - } - key := fmt.Sprint(info.ModTime().UnixNano(), info.Size(), info.Mode()) - if st, ok := info.Sys().(*syscall.Stat_t); ok { - key += fmt.Sprint(" ", st.Ino, st.Ctim.Nano()) - } - b.read.mu.Lock() - defer b.read.mu.Unlock() - if b.read.key == key { - return b.read.keep, b.read.err - } raw, err := os.ReadFile(path) - if errors.Is(err, fs.ErrPermission) { - var out string - out, err = b.Run(ctx, "cat", "--", path) - raw = []byte(out) - } - if err != nil { - // Not kept: a read that failed may succeed at the next look. - return Retention{}, fmt.Errorf("%s cannot be read: %w", path, err) + switch { + case errors.Is(err, fs.ErrNotExist): + return Retention{}, fmt.Errorf("%s %s", path, Missing) + case errors.Is(err, fs.ErrPermission): + return Retention{}, fmt.Errorf("%s %s (%v)", path, Unreadable, errors.Unwrap(err)) + case err != nil: + return Retention{}, err } keep, err := ParseRetention(string(raw)) if err != nil { - err = fmt.Errorf("%s: %w", path, err) + return Retention{}, fmt.Errorf("%s: %w", path, err) } - b.read.key, b.read.keep, b.read.err = key, keep, err - return keep, err + return keep, nil } // InForce is the retention a night thins by, or why it thins nothing: what `backed-up` says beside each diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index 7449ba30..9b489b80 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -69,23 +69,18 @@ func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) { } } -// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339), and the night runs as the -// operator account: what the account cannot read is read as root, as every other root's file here is -// (novox/hq issue 454). Before, every machine's night refused it as unreadable and thinned nothing. -func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) { - where := placed(t, "# pg\npath /\n") - path := filepath.Join(where.State, RetentionFile) - must(t, os.WriteFile(path, []byte("daily=7\nweekly=4\nmonthly=24\n"), 0o600)) - must(t, os.Chmod(path, 0)) +// A file the account the night runs as may not read — the mode an older build of this module had the mesh +// write it in (0600, root's) — thins nothing, and says why (novox/hq issue 454). +func TestARetentionTheAccountCannotReadThinsNothingAndSaysWhy(t *testing.T) { if os.Getuid() == 0 { - t.Skip("root reads a file of mode 0 itself; nothing to escalate") + t.Skip("root reads a file of mode 0 itself") } - forget, catted := "", "" + where := placed(t, "# pg\npath /\n") + must(t, os.Chmod(filepath.Join(where.State, RetentionFile), 0)) + forget := "" + var said []string run := func(_ context.Context, name string, args ...string) (string, error) { switch { - case name == "cat": - catted = strings.Join(args, " ") - return "daily=7\nweekly=4\nmonthly=24\n", nil case name == "restic" && args[5] == "forget": forget = "restic " + strings.Join(args[5:], " ") case name == "restic" && args[5] == "backup": @@ -93,14 +88,44 @@ func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) { } return "", nil } - b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet} + b := &Backups{Where: where, Run: run, Now: time.Now, Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} _, err := b.BackUp(context.Background(), "") must(t, err) - if catted != "-- "+path { - t.Errorf("read %q as root, want %q", catted, "-- "+path) + if forget != "" { + t.Errorf("thinned by a file it could not read: %q", forget) } - if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; forget != want { - t.Errorf("ran %q, want %q", forget, want) + if all := strings.Join(said, "\n"); !strings.Contains(all, Unreadable) || !strings.Contains(all, "every restore point is kept") { + t.Errorf("said %q", all) + } + h := b.Health(context.Background()) + if h.State != "unhealthy" || !strings.Contains(h.Why, Unreadable) || !strings.Contains(h.Why, "0644") { + t.Errorf("%s: %q, want it unhealthy naming the cause", h.State, h.Why) + } +} + +// The mesh writes the file readable, and root's: the account reads it and cannot change it. +func TestTheManifestWritesTheRetentionReadable(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + must(t, err) + var m struct { + Resources []map[string]any `json:"resources"` + } + must(t, json.Unmarshal(raw, &m)) + file, night := -1, -1 + for i, r := range m.Resources { + switch r["id"] { + case "retention": + file = i + if r["mode"] != "0644" || r["owner"] != nil { + t.Errorf("the retention file is mode %v owner %v, want 0644 and root's", r["mode"], r["owner"]) + } + case "night": + night = i + } + } + // Applied in declared order: the rollout that brings the night's process makes the file readable first. + if file < 0 || night < 0 || file > night { + t.Errorf("the retention file is declared at %d and the night at %d", file, night) } } diff --git a/modules/restic/module.json b/modules/restic/module.json index 57e93053..544c0b81 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -84,7 +84,7 @@ "id": "retention", "type": "file", "path": "${dir:state}/retention.conf", - "mode": "0600", + "mode": "0644", "trusted": true, "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly and keep-monthly (novox/hq issue 416). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\n" },