Merge pull request 'restic: a failed backup, a failed thinning and a refused retention raise a condition (hq issue 454)' (#210) from fix/454-a-failed-backup-is-raised into main

modules/restic/cmd/restic-backups/awake.go

@@ -0,0 +1,102 @@
+package main
+
+// A night that never happens (novox/hq issue 454): the night's process hangs, its timer never fires, a
+// restore or a deletion holds the lock for hours, or the lock cannot be taken. None of that leaves a failed
+// night behind, so the health also asks how long this machine has been up, with the night's process running,
+// since the newest good night. Up, not since: a laptop asleep at the hour, or a machine switched off, has
+// missed nothing it could have done, and the night's process runs one soon after it starts again.
+//
+// The process keeps the count itself, once a minute, from Go's monotonic clock, which on Linux stops while
+// the machine is suspended; a gap between two looks is counted only as long as the machine was awake. Time
+// the process was not running is not counted either: that is its unit's to say, judged by liveness.
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// Overdue is how long a machine may be up without a good night before it is said: a day, and the two hours
+// a night may take or be late by.
+const Overdue = 26 * time.Hour
+
+// LookEvery is how often the night's process counts; Stale how old its count may be before it is said.
+const (
+	LookEvery = time.Minute
+	Stale     = 10 * time.Minute
+)
+
+// Awake is the night's process's count: since which good night (zero before the first), how long the
+// machine was up with the process running, when the process started and when it last counted.
+type Awake struct {
+	From    time.Time     `json:"from"`
+	Up      time.Duration `json:"up"`
+	Started time.Time     `json:"started"`
+	Looked  time.Time     `json:"looked"`
+}
+
+func (b *Backups) awakeFile() string { return filepath.Join(b.Where.State, "awake.json") }
+
+// LastAwake is the count as the night's process last wrote it; nil before it ever did.
+func (b *Backups) LastAwake() *Awake {
+	raw, err := os.ReadFile(b.awakeFile())
+	if err != nil {
+		return nil
+	}
+	var a Awake
+	if json.Unmarshal(raw, &a) != nil {
+		return nil
+	}
+	return &a
+}
+
+func (b *Backups) keepAwake(a Awake) {
+	raw, _ := json.MarshalIndent(a, "", "  ")
+	if err := os.WriteFile(b.awakeFile(), append(raw, '\n'), 0o600); err != nil {
+		b.Say("recording how long this machine was up since its last good night failed: %v", err)
+	}
+}
+
+// newestGood is the newest good night of any module; zero before the first.
+func newestGood(nights map[string]Night) time.Time {
+	var newest time.Time
+	for _, n := range nights {
+		if n.OK && n.At.After(newest) {
+			newest = n.At
+		}
+	}
+	return newest
+}
+
+// Count adds one look: up is how long the machine was awake since the last one. A newer good night starts
+// the count again. Pure.
+func (a Awake) Count(good time.Time, up time.Duration, now time.Time) Awake {
+	if !good.Equal(a.From) {
+		a.From, a.Up = good, 0
+	}
+	if up > 0 {
+		a.Up += up
+	}
+	a.Looked = now.UTC()
+	return a
+}
+
+// counting is the night's process's count, once a minute while it runs. It carries on from what an earlier
+// process left, so a restart neither forgives an overdue night nor starts one fresh.
+func counting(b *Backups) {
+	a := Awake{}
+	if was := b.LastAwake(); was != nil {
+		a = *was
+	}
+	a.Started = time.Now().UTC()
+	last := time.Now()
+	for {
+		now := time.Now()
+		// Both readings carry the monotonic clock, which does not run while the machine is suspended.
+		a = a.Count(newestGood(b.Nights()), now.Sub(last), now)
+		last = now
+		b.keepAwake(a)
+		time.Sleep(LookEvery)
+	}
+}

modules/restic/cmd/restic-backups/backups.go

@@ -9,9 +9,9 @@
 //
 // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
 // restored on its own. Everything lands in one repository on the machine — deduplicated, so every
-// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
-// 8 weekly and 6 monthly unless the machine's settings say otherwise (retention.go). Against
-// mistakes, not disasters: nothing leaves the machine.
+// night is a complete restore point and only what changed costs space — and is thinned by the machine's
+// settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters:
+// nothing leaves the machine. How the last night and thinning went is the module's health (health.go).
 //
 // Root's: the dumps read every store and the repository holds every module's data, and the runtime
 // launching this binary runs as the operator's account, so restic and the run lines go through sudo
@@ -32,7 +32,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
-	"sync"
+	"syscall"
 	"time"
 )
 
@@ -178,13 +178,43 @@ func whereFromEnv() (Where, error) {
 }
 
 // Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never
-// share a dump.
+// share a dump — across the night's process and the tool runtime alike (hold).
 type Backups struct {
 	Where Where
 	Run   Runner
 	Now   func() time.Time
 	Say   func(format string, args ...any)
-	mu    sync.Mutex
+}
+
+// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a
+// retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer
+// keeps the other out.
+const (
+	nightLock   = "night.lock"
+	measureLock = "measure.lock"
+)
+
+// hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open
+// file's, so two holders in one process exclude each other as two processes do.
+func (b *Backups) hold(name string) (func(), error) {
+	f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err)
+	}
+	for {
+		err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX)
+		if !errors.Is(err, syscall.EINTR) {
+			break
+		}
+	}
+	if err != nil {
+		f.Close()
+		return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err)
+	}
+	return func() {
+		_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
 }
 
 // exists is whether a path is there, asked as root: a store's directory is often its own user's
@@ -299,16 +329,27 @@ func snapshotOf(out string) string {
 	return id
 }
 
-// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome.
+// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and
+// keeps how the night as a whole and its thinning went for the health (novox/hq issue 454).
 func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if err := b.ensureRepository(ctx); err != nil {
+	run := b.LastRun()
+	run.At = b.Now().UTC()
+	failed := func(err error) (map[string]Night, error) {
+		run.Failed = err.Error()
+		b.keepRun(run)
 		return nil, err
 	}
+	release, err := b.hold(nightLock)
+	if err != nil {
+		return failed(err)
+	}
+	defer release()
+	if err := b.ensureRepository(ctx); err != nil {
+		return failed(err)
+	}
 	all, err := b.Declared()
 	if err != nil {
-		return nil, err
+		return failed(err)
 	}
 	chosen := all
 	if only != "" {
@@ -321,6 +362,7 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er
 			}
 		}
 		if len(chosen) == 0 {
+			// Asked of one module that is not here: nothing ran, and the last night stands as it was.
 			return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names))
 		}
 	}
@@ -331,16 +373,59 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er
 	}
 	// A retention that does not read thins nothing: keeping too much costs space, thinning by a value
 	// nobody meant loses restore points (novox/hq issue 416).
-	keep, err := b.retention()
+	thinned := Thinning{At: b.Now().UTC()}
+	keep, err := b.retention(ctx)
 	if err != nil {
+		thinned.Refused = err.Error()
 		b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err)
 	} else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
 		"--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil {
+		thinned.Kept, thinned.Error = &keep, err.Error()
 		b.Say("thinning the restore points failed, and every one is kept: %v", err)
+	} else {
+		thinned.Kept, thinned.OK = &keep, true
 	}
+	run.Failed, run.Thinned = "", &thinned
+	b.keepRun(run)
 	return outcome, nil
 }
 
+// Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its
+// thinning. Each module's own outcome is its Night.
+type Run struct {
+	At time.Time `json:"at"`
+	// Failed is why it did not run at all: the repository would not open, or what to back up does not read.
+	Failed  string    `json:"failed,omitempty"`
+	Thinned *Thinning `json:"thinned,omitempty"`
+}
+
+// Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing.
+type Thinning struct {
+	At      time.Time  `json:"at"`
+	OK      bool       `json:"ok"`
+	Kept    *Retention `json:"kept,omitempty"`
+	Refused string     `json:"refused,omitempty"`
+	Error   string     `json:"error,omitempty"`
+}
+
+func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") }
+
+// LastRun is how the last night went as a whole; empty before the first.
+func (b *Backups) LastRun() Run {
+	var r Run
+	if raw, err := os.ReadFile(b.runFile()); err == nil {
+		_ = json.Unmarshal(raw, &r)
+	}
+	return r
+}
+
+func (b *Backups) keepRun(r Run) {
+	raw, _ := json.MarshalIndent(r, "", "  ")
+	if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil {
+		b.Say("recording how the night went failed: %v", err)
+	}
+}
+
 func orNothing(names []string) string {
 	if len(names) == 0 {
 		return "nothing"
@@ -376,6 +461,11 @@ type ModuleBackups struct {
 	LastNight     *Night    `json:"lastNight"`
 	RestorePoints int       `json:"restorePoints"`
 	Newest        *Snapshot `json:"newest,omitempty"`
+	// Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned
+	// how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its
+	// own, so the answer stays the list the controller reads.
+	Retention InForce   `json:"retention"`
+	Thinned   *Thinning `json:"thinned,omitempty"`
 	// Data is every item the module declares, measured, with its newest good backup (novox/hq ADR
 	// 0233): what the controller's self-check reads.
 	Data []ItemReport `json:"data"`
@@ -396,6 +486,8 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups,
 	good := b.Good()
 	measured := b.Measurements()
 	snaps, _ := b.Snapshots(ctx, module)
+	inForce := b.inForce(ctx)
+	thinned := b.LastRun().Thinned
 	out := []ModuleBackups{}
 	listed := map[string]bool{}
 	for _, d := range declared {
@@ -403,7 +495,7 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups,
 			continue
 		}
 		listed[d.Module] = true
-		m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths,
+		m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned,
 			Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])}
 		if n, ok := nights[d.Module]; ok {
 			m.LastNight = &n
@@ -440,8 +532,11 @@ type Restored struct {
 // state as restored/<stamp>/<module>/<path> (besideIsUnsafe). A target that already exists is refused, never
 // overwritten.
 func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
+	release, err := b.hold(nightLock)
+	if err != nil {
+		return nil, err
+	}
+	defer release()
 	mine, err := b.Snapshots(ctx, module)
 	if err != nil {
 		return nil, err
@@ -583,9 +678,12 @@ func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) {
 
 // Check is the weekly look at the repository's own integrity, with a sample of the data read back.
 func (b *Backups) Check(ctx context.Context) error {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	_, err := b.restic(ctx, "check", "--read-data-subset", "5%")
+	release, err := b.hold(nightLock)
+	if err != nil {
+		return err
+	}
+	defer release()
+	_, err = b.restic(ctx, "check", "--read-data-subset", "5%")
 	return err
 }
 
@@ -602,11 +700,6 @@ func nextNight(now time.Time, hour int) time.Time {
 // missedANight is whether a night was missed: the newest good night of any module is older than a
 // day and a bit — the machine was off, or this module was not running, at the hour.
 func missedANight(nights map[string]Night, now time.Time) bool {
-	var newest time.Time
-	for _, n := range nights {
-		if n.OK && n.At.After(newest) {
-			newest = n.At
-		}
-	}
-	return newest.IsZero() || now.Sub(newest) > 26*time.Hour
+	newest := newestGood(nights)
+	return newest.IsZero() || now.Sub(newest) > Overdue
 }

modules/restic/cmd/restic-backups/backups_test.go

@@ -25,6 +25,8 @@ func placed(t *testing.T, declared string) Where {
 	dir := t.TempDir()
 	must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600))
 	must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600))
+	// The retention file every machine has, as the mesh writes it with the defaults (novox/hq issue 454).
+	must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600))
 	return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"),
 		PasswordFile: filepath.Join(dir, "pw"), State: dir}
 }

modules/restic/cmd/restic-backups/data.go

@@ -157,8 +157,6 @@ func (b *Backups) Measurements() map[string]map[string]Measured {
 	return out
 }
 
-var measuring sync.Mutex
-
 // The ways an item is measured.
 const (
 	measureWalk    = "walk"
@@ -364,8 +362,12 @@ func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
 // measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the
 // hourly round reads counters, and a walk runs at most once a day.
 func (b *Backups) MeasureAll(ctx context.Context, walks bool) error {
-	measuring.Lock()
-	defer measuring.Unlock()
+	// The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454).
+	release, err := b.hold(measureLock)
+	if err != nil {
+		return err
+	}
+	defer release()
 	items, err := b.Items()
 	if err != nil {
 		return err
@@ -552,8 +554,13 @@ func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm
 }
 
 func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
+	release, err := b.hold(nightLock)
+	if err != nil {
+		d.Running, d.Done, d.Error = false, true, err.Error()
+		b.keepDeletion(path, d)
+		return
+	}
+	defer release()
 	finish := func(err error) {
 		d.Running, d.Done = false, true
 		if err != nil {

modules/restic/cmd/restic-backups/health.go

@@ -0,0 +1,189 @@
+package main
+
+// The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be
+// loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines
+// only — a typo in a setting let the repository grow until a disk condition fired.
+//
+// Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run
+// at all, when no night has backed anything up while this machine was up for more than a day (awake.go), when
+// the night's process has stopped counting, when the last thinning failed, or when the retention file is
+// missing, cannot be read or is not in the form the mesh writes. A retention setting the night refuses is the operator's to give again: the
+// answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator
+// condition. The node-engine asks this through the node tools for the night's process; the controller
+// raises module.restic.<machine>.unhealthy on the second look that says so, with these words.
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why,
+// waits), with what was found and what was said beside it.
+type Health struct {
+	health.Answer
+	Found []string `json:"found,omitempty"`
+	Said  []string `json:"said,omitempty"`
+}
+
+// Seen is what the health is judged from.
+type Seen struct {
+	Declared    []Declared
+	DeclaredErr error
+	Nights      map[string]Night
+	Good        map[string]time.Time
+	Run         Run
+	Retention   InForce
+	// Awake is the night's process's count of how long this machine was up since its newest good night; nil
+	// before the process ever counted.
+	Awake *Awake
+	Now   time.Time
+}
+
+func hours(d time.Duration) string {
+	if d < 2*time.Hour {
+		return fmt.Sprintf("%d minutes", int(d.Minutes()))
+	}
+	return fmt.Sprintf("%d hours", int(d.Hours()))
+}
+
+func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") }
+
+// Judge is the health of what was seen. Pure.
+func Judge(s Seen) Health {
+	var found, said []string
+	var waits []health.Wait
+	if s.DeclaredErr != nil {
+		found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+
+			"back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr))
+	}
+	if s.Run.Failed != "" {
+		found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+
+			"names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed))
+	}
+	if a := s.Awake; a != nil {
+		newest := "no night has backed anything up here yet"
+		if !a.From.IsZero() {
+			newest = "the newest good night was at " + clock(a.From)
+		}
+		switch {
+		case s.Now.Sub(a.Looked) > Stale:
+			found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+
+				"night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+
+				"on this machine.", clock(a.Looked), hours(Stale), newest))
+		case a.Up > Overdue:
+			found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+
+				"night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+
+				"the repository, or never starts leaves no failed night behind; read the journal of restic-night "+
+				"(node-service-manager.journal), then take one with node-backup.now on this machine.", newest, hours(a.Up)))
+		default:
+			said = append(said, fmt.Sprintf("Up %s with the night's process running since the newest good night; "+
+				"said when that passes %s.", hours(a.Up), hours(Overdue)))
+		}
+	}
+	failed := map[string]Night{}
+	for _, d := range s.Declared {
+		if n, ok := s.Nights[d.Module]; ok && !n.OK {
+			failed[d.Module] = n
+		}
+	}
+	names := make([]string, 0, len(failed))
+	for m := range failed {
+		names = append(names, m)
+	}
+	sort.Strings(names)
+	for _, m := range names {
+		n := failed[m]
+		since := "it has no restore point here"
+		if g, ok := s.Good[m]; ok && !g.IsZero() {
+			since = "its newest restore point is of " + clock(g)
+		}
+		found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+
+			`with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m))
+	}
+	if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" {
+		found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. Nothing was forgotten and every "+
+			"restore point is kept, so the repository grows until a thinning succeeds; fix what the error names, then "+
+			"node-backup.now on this machine thins again.", clock(t.At), strings.TrimSuffix(t.Error, ".")))
+	}
+	r := s.Retention
+	switch {
+	case r.Refused == "" && r.Retention != nil:
+		said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module.",
+			r.Daily, r.Weekly, r.Monthly))
+	case r.Setting != "":
+		value := "not set"
+		if r.Value != "" {
+			value = fmt.Sprintf("%q", r.Value)
+		}
+		line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+
+			"nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+
+			"(mesh-controller.settings).", r.Setting, value, r.Setting)
+		said = append(said, line)
+		waits = append(waits, health.Wait{Part: "night", Setting: r.Setting,
+			What: "a count from 1 to 9999 for " + r.Setting})
+	case strings.Contains(r.Refused, Unreadable):
+		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
+			"kept, so the repository grows. The mesh writes the file readable (0644) and root's; one the account cannot "+
+			"read was written by an older build of this module, or changed on the machine, and the next push of this "+
+			"machine (mesh-controller.push) writes it readable again.", r.Refused))
+	case strings.HasSuffix(r.Refused, Missing):
+		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
+			"kept until it is there again, rather than thinned by a number nobody set; push this machine "+
+			"(mesh-controller.push) so the mesh writes it again.", r.Refused))
+	case strings.Contains(r.Refused, RetentionFile+": "):
+		// Read, and not in the form the mesh writes: something other than the mesh wrote it.
+		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
+			"kept, so the repository grows. The mesh writes only daily=, weekly= and monthly=, so something else wrote "+
+			"this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused))
+	default:
+		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
+			"kept, so the repository grows. Fix what the error names.", r.Refused))
+	}
+	if len(found) > 0 {
+		why := strings.Join(found, " ")
+		if len(waits) > 0 {
+			why += " " + said[len(said)-1]
+		}
+		return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said}
+	}
+	if len(waits) > 0 {
+		return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said}
+	}
+	return Health{Answer: health.Healthy(healthySaid(s, said)), Said: said}
+}
+
+// healthySaid is what was seen, and only that, when nothing is wrong.
+func healthySaid(s Seen, said []string) string {
+	var parts []string
+	switch {
+	case len(s.Declared) == 0:
+		parts = append(parts, "No module here declares anything to back up.")
+	case newestGood(s.Nights).IsZero():
+		parts = append(parts, "No good night is recorded here yet.")
+	default:
+		parts = append(parts, fmt.Sprintf("No module declared here failed its last night; the newest good night was at %s.",
+			clock(newestGood(s.Nights))))
+	}
+	switch t := s.Run.Thinned; {
+	case t == nil:
+		parts = append(parts, "No thinning is recorded yet.")
+	case t.OK && t.Kept != nil:
+		parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points.",
+			clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly))
+	case t.Refused != "":
+		parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At)))
+	}
+	return strings.Join(append(parts, said...), " ")
+}
+
+// Health reads what the nights left and the retention as it is now, and judges them.
+func (b *Backups) Health(ctx context.Context) Health {
+	declared, err := b.Declared()
+	return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(),
+		Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()})
+}

modules/restic/cmd/restic-backups/health_test.go

@@ -0,0 +1,382 @@
+package main
+
+// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and
+// a refused retention each raise a condition naming the module and what to do, instead of a journal line
+// nobody reads until a disk fills.
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails
+// with thinning when it is not empty.
+type nightly struct {
+	failing  map[string]bool
+	thinning string
+}
+
+func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) {
+	switch {
+	case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]:
+		return "", errors.New("the dump failed: no space left on device")
+	case name == "restic" && args[5] == "backup":
+		return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
+	case name == "restic" && args[5] == "forget" && n.thinning != "":
+		return "", errors.New(n.thinning)
+	}
+	return "", nil
+}
+
+func nightAt(t *testing.T) (*Backups, *nightly) {
+	t.Helper()
+	n := &nightly{failing: map[string]bool{}}
+	b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run,
+		Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet}
+	return b, n
+}
+
+func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) {
+	b, n := nightAt(t)
+	n.failing["mail"] = true
+	_, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	h := b.Health(context.Background())
+	if h.State != health.StateUnhealthy {
+		t.Fatalf("a failed backup is %s: %s", h.State, h.Why)
+	}
+	for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} {
+		if !strings.Contains(h.Why, want) {
+			t.Errorf("said %q, want it to hold %q", h.Why, want)
+		}
+	}
+	if strings.Contains(h.Why, "pg was NOT") {
+		t.Errorf("pg was backed up and is said as failed: %q", h.Why)
+	}
+
+	// Backed up again, it clears.
+	n.failing["mail"] = false
+	_, err = b.BackUp(context.Background(), "mail")
+	must(t, err)
+	if h := b.Health(context.Background()); h.State != health.StateHealthy {
+		t.Fatalf("backed up again, it is %s: %s", h.State, h.Why)
+	}
+}
+
+func TestANightThatDidNotRunIsUnhealthy(t *testing.T) {
+	b, _ := nightAt(t)
+	b.Run = func(_ context.Context, name string, args ...string) (string, error) {
+		return "", errors.New("Fatal: wrong password or no key found")
+	}
+	if _, err := b.BackUp(context.Background(), ""); err == nil {
+		t.Fatal("a repository that will not open ran a night")
+	}
+	h := b.Health(context.Background())
+	if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") {
+		t.Fatalf("a night that did not run: %s: %q", h.State, h.Why)
+	}
+}
+
+func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) {
+	b, n := nightAt(t)
+	n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively"
+	_, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	h := b.Health(context.Background())
+	if h.State != health.StateUnhealthy {
+		t.Fatalf("a failed thinning is %s: %s", h.State, h.Why)
+	}
+	for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} {
+		if !strings.Contains(h.Why, want) {
+			t.Errorf("said %q, want it to hold %q", h.Why, want)
+		}
+	}
+	n.thinning = ""
+	_, err = b.BackUp(context.Background(), "")
+	must(t, err)
+	if h := b.Health(context.Background()); h.State != health.StateHealthy {
+		t.Fatalf("thinned again, it is %s: %s", h.State, h.Why)
+	}
+}
+
+// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283),
+// and the controller says it as the module's needs-operator condition with the act.
+func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) {
+	b, _ := nightAt(t)
+	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600))
+	h := b.Health(context.Background())
+	if h.State != health.StateWaits {
+		t.Fatalf("a refused setting is %s: %s", h.State, h.Why)
+	}
+	if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil {
+		t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits)
+	}
+	for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} {
+		if !strings.Contains(h.Why, want) {
+			t.Errorf("said %q, want it to hold %q", h.Why, want)
+		}
+	}
+}
+
+// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the
+// operator gave: it is a fault, and says to push the machine.
+func TestAMissingRetentionFileIsUnhealthy(t *testing.T) {
+	b, _ := nightAt(t)
+	must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile)))
+	h := b.Health(context.Background())
+	if h.State != health.StateUnhealthy {
+		t.Fatalf("a missing retention file is %s: %s", h.State, h.Why)
+	}
+	for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} {
+		if !strings.Contains(h.Why, want) {
+			t.Errorf("said %q, want it to hold %q", h.Why, want)
+		}
+	}
+}
+
+// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for.
+func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) {
+	b, n := nightAt(t)
+	n.failing["pg"] = true
+	_, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600))
+	h := b.Health(context.Background())
+	if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") {
+		t.Fatalf("%s: %q", h.State, h.Why)
+	}
+}
+
+func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) {
+	b, _ := nightAt(t)
+	if h := b.Health(context.Background()); h.State != health.StateHealthy {
+		t.Fatalf("before any night: %s: %s", h.State, h.Why)
+	}
+}
+
+// A module no longer declared here is not held against the machine for a night it failed before.
+func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) {
+	b, n := nightAt(t)
+	n.failing["mail"] = true
+	_, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600))
+	if h := b.Health(context.Background()); h.State != health.StateHealthy {
+		t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why)
+	}
+}
+
+// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning.
+func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) {
+	b, n := nightAt(t)
+	n.thinning = "Fatal: repository is already locked"
+	_, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	listed, err := b.BackedUp(context.Background(), "pg")
+	must(t, err)
+	if len(listed) != 1 {
+		t.Fatalf("listed %+v", listed)
+	}
+	raw, _ := json.Marshal(listed[0])
+	var got struct {
+		Retention map[string]any `json:"retention"`
+		Thinned   map[string]any `json:"thinned"`
+	}
+	must(t, json.Unmarshal(raw, &got))
+	if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil {
+		t.Errorf("retention %v, want 14/8/6 not refused", got.Retention)
+	}
+	if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") {
+		t.Errorf("thinned %v, want the failure", got.Thinned)
+	}
+
+	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600))
+	listed, err = b.BackedUp(context.Background(), "pg")
+	must(t, err)
+	raw, _ = json.Marshal(listed[0])
+	got.Retention = nil
+	must(t, json.Unmarshal(raw, &got))
+	if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" {
+		t.Errorf("retention %v, want it refused naming keep-daily", got.Retention)
+	}
+}
+
+// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at
+// a time across both, never two nights or a night and a restore over one repository.
+func TestANightWaitsForAnotherProcesssNight(t *testing.T) {
+	b, _ := nightAt(t)
+	release, err := b.hold(nightLock)
+	must(t, err)
+	done := make(chan struct{})
+	go func() {
+		other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet}
+		_, _ = other.BackUp(context.Background(), "")
+		close(done)
+	}()
+	select {
+	case <-done:
+		t.Fatal("a night ran while another held the repository")
+	case <-time.After(200 * time.Millisecond):
+	}
+	release()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the night did not run once the other let go")
+	}
+}
+
+// The manifest runs the night and the measuring as processes of their own, as the operator account like the
+// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as
+// before), each given where everything is; the night judged by backup_health, the measuring by its unit —
+// a tool check only beside a check of another kind (ADR 0240 rule 2).
+func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
+	raw, err := os.ReadFile("../../module.json")
+	must(t, err)
+	var m struct {
+		Tools     []string         `json:"tools"`
+		Resources []map[string]any `json:"resources"`
+		Build     struct {
+			Artifacts []struct {
+				Name string            `json:"name"`
+				Env  map[string]string `json:"env"`
+			} `json:"artifacts"`
+		} `json:"build"`
+	}
+	must(t, json.Unmarshal(raw, &m))
+	served := map[string]bool{}
+	for _, tool := range tools(&Backups{}) {
+		served[tool.Name] = true
+	}
+	for _, tool := range m.Tools {
+		if !served[tool] {
+			t.Errorf("the manifest lists %s, which the bundle does not serve", tool)
+		}
+	}
+	env := m.Build.Artifacts[0].Env
+	want := map[string]struct{ run, kind, tool string }{
+		"night":   {"night", "tool", "backup_health"},
+		"measure": {"measure", "unit", ""},
+	}
+	for _, r := range m.Resources {
+		w, ok := want[r["id"].(string)]
+		if !ok || r["type"] != "process" {
+			continue
+		}
+		delete(want, r["id"].(string))
+		run, _ := json.Marshal(r["run"])
+		if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name {
+			t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"])
+		}
+		if r["user"] != "${machine:account}" {
+			t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"])
+		}
+		penv, _ := r["env"].(map[string]any)
+		for k, v := range env {
+			if penv[k] != v {
+				t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v)
+			}
+		}
+		h, _ := r["health"].(map[string]any)
+		if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) {
+			t.Errorf("%s is judged by %v", r["id"], h)
+		}
+		if w.tool != "" && !slicesContains(m.Tools, w.tool) {
+			t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool)
+		}
+	}
+	for id := range want {
+		t.Errorf("no process %s", id)
+	}
+}
+
+func slicesContains(s []string, v string) bool {
+	for _, x := range s {
+		if x == v {
+			return true
+		}
+	}
+	return false
+}
+
+// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a
+// day since the newest good night, counted only while it was awake (novox/hq issue 454).
+func TestANightThatNeverHappensIsUnhealthy(t *testing.T) {
+	good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
+	now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC)
+	seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
+		Nights:    map[string]Night{"pg": {OK: true, At: good}},
+		Retention: InForce{Retention: &TodaysRetention}, Now: now}
+
+	seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}
+	h := Judge(seen)
+	if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") {
+		t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why)
+	}
+
+	// A laptop asleep most of those two days: up 20 hours of them, nothing is overdue.
+	seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)}
+	if h := Judge(seen); h.State != health.StateHealthy {
+		t.Fatalf("up 20 hours: %s: %q", h.State, h.Why)
+	}
+
+	// The process stopped counting: said, not taken for a night that is on time.
+	seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)}
+	if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
+		t.Fatalf("a count an hour old: %s: %q", h.State, h.Why)
+	}
+}
+
+func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) {
+	first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
+	now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC)
+	a := Awake{}.Count(first, time.Minute, now)
+	a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake
+	if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) {
+		t.Fatalf("counted %+v", a)
+	}
+	a = a.Count(first.Add(24*time.Hour), time.Minute, now)
+	if a.Up != time.Minute {
+		t.Fatalf("a newer good night did not start the count again: %+v", a)
+	}
+}
+
+// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one
+// called a success.
+func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) {
+	good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC)
+	seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
+		Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good}
+	h := Judge(seen)
+	if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") {
+		t.Fatalf("with no thinning recorded: %q", h.Why)
+	}
+	seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""}
+	if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") {
+		t.Fatalf("after a refused thinning: %q", h.Why)
+	}
+	seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24}}
+	if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") {
+		t.Fatalf("after a thinning: %q", h.Why)
+	}
+}
+
+// The lock that cannot be taken is a night that did not run, and recorded as one.
+func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) {
+	b, _ := nightAt(t)
+	must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write
+	if _, err := b.BackUp(context.Background(), ""); err == nil {
+		t.Fatal("a night ran without its lock")
+	}
+	if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") {
+		t.Fatalf("%s: %q", h.State, h.Why)
+	}
+}

modules/restic/cmd/restic-backups/main.go

@@ -1,7 +1,16 @@
-// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the
-// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It
-// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the
-// live data — and, beside them, runs the night that happens without anyone asking.
+// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, three ways to run:
+//
+//   - with no argument, launched by the machine's tool runtime and speaking MCP to it over stdio through the
+//     Go SDK (ADR 0193, ADR 0198): the node-backup seat's three verbs — what is backed up, take one now,
+//     restore beside the live data — and the health check, backup_health;
+//   - `night`, a process the node-engine runs: the night that happens without anyone asking, judged by
+//     backup_health from what it leaves;
+//   - `measure`, a process the node-engine runs: the hourly measuring of the data declared here (ADR 0233),
+//     judged by its unit.
+//
+// The night and the measuring were goroutines of the tool runtime's process until novox/hq issue 454: a module
+// raises a condition only through the health of something it runs (ADR 0240, ADR 0256), and a check by the
+// module's own tool only beside a check of another kind, so each runs as a process of its own.
 //
 // stdout is the MCP channel; everything this module says, it says on stderr.
 package main
@@ -31,11 +40,24 @@ func main() {
 		os.Exit(1)
 	}
 	b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
-	go nights(b)
-	go measurements(b)
-	if err := stdio.Serve("", tools(b)); err != nil {
-		say("%v", err)
-		os.Exit(1)
+	way := ""
+	if len(os.Args) > 1 {
+		way = os.Args[1]
+	}
+	switch way {
+	case "night":
+		nights(b)
+	case "measure":
+		measurements(b)
+	case "":
+		if err := stdio.Serve("", tools(b)); err != nil {
+			say("%v", err)
+			os.Exit(1)
+		}
+	default:
+		say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+
+			"measure it measures the data declared here", way)
+		os.Exit(2)
 	}
 }
 
@@ -47,6 +69,7 @@ func nights(b *Backups) {
 	if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
 		hour = h
 	}
+	go counting(b)
 	if missedANight(b.Nights(), time.Now()) {
 		time.Sleep(10 * time.Minute)
 		night(b)
@@ -154,6 +177,11 @@ func tools(b *Backups) []stdio.Tool {
 				}
 				return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
 			}),
+		{Name: "backup_health", Description: "Whether this machine's backups hold, as the module's health check " +
+			"(novox/hq ADR 0240, issue 454): unhealthy when a module was NOT backed up at its last night, the last night " +
+			"did not run, the last thinning failed, or the retention file is missing or unreadable — each said with what " +
+			"to do; waits, naming the setting, when a retention setting is refused. (r)",
+			Run: func(map[string]any) (any, error) { return b.Health(context.Background()), nil }},
 		// Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup
 		// delete` asks these. Not seat verbs — nobody else calls them.
 		{Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " +

modules/restic/cmd/restic-backups/retention.go

@@ -7,8 +7,10 @@ package main
 // refused here and never handed to restic (issue 346: a raw setting is an injection point).
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -17,10 +19,14 @@ import (
 )
 
 // Retention is how many restore points each period keeps.
-type Retention struct{ Daily, Weekly, Monthly int }
+type Retention struct {
+	Daily   int `json:"daily"`
+	Weekly  int `json:"weekly"`
+	Monthly int `json:"monthly"`
+}
 
 // TodaysRetention is the rotation before it was a preference (novox/hq ADR 0214), and still the
-// manifest's default.
+// manifest's default — never a fallback for a file that is not there (novox/hq issue 454).
 var TodaysRetention = Retention{14, 8, 6}
 
 // RetentionFile is the file the mesh writes the retention into, in this module's state.
@@ -52,32 +58,80 @@ func ParseRetention(text string) (Retention, error) {
 		case seen[k] != 0:
 			return Retention{}, fmt.Errorf("%s is given twice", k)
 		case !count.MatchString(v):
-			return Retention{}, fmt.Errorf("the %s setting is %q; it is a count from 1 to 9999", setting, v)
+			return Retention{}, &SettingRefused{Setting: setting, Value: v}
 		}
 		seen[k], _ = strconv.Atoi(v)
 	}
 	for _, k := range []string{"daily", "weekly", "monthly"} {
 		if seen[k] == 0 {
-			return Retention{}, fmt.Errorf("the %s setting is missing", settingOf[k])
+			return Retention{}, &SettingRefused{Setting: settingOf[k]}
 		}
 	}
 	return Retention{seen["daily"], seen["weekly"], seen["monthly"]}, nil
 }
 
-// retention is this machine's: today's where the module was placed by a definition older than the
-// file, else what the file says.
-func (b *Backups) retention() (Retention, error) {
+// SettingRefused is one retention setting the night refuses: the operator's to give again.
+type SettingRefused struct{ Setting, Value string }
+
+func (e *SettingRefused) Error() string {
+	if e.Value == "" {
+		return fmt.Sprintf("the %s setting is missing", e.Setting)
+	}
+	return fmt.Sprintf("the %s setting is %q; it is a count from 1 to 9999", e.Setting, e.Value)
+}
+
+// Missing is said of a retention file that is not there. Every machine has had the file since mesh-catalog
+// #200 rolled out (seen in each machine's plan, 2026-10-11), so a missing one is lost, not old: it is refused
+// rather than thinned by 14/8/6, which would thin by a number nobody set (novox/hq issue 454).
+const Missing = "is missing"
+
+// Unreadable is said of a retention file this program's account may not read.
+const Unreadable = "cannot be read by the account this module runs as"
+
+// retention is this machine's, as the file the mesh wrote says; an error when it is missing, cannot be
+// read, or does not read, and a *SettingRefused within it when one setting is what is wrong.
+//
+// The mesh writes the file root's and readable by everyone (0644): the account the night runs as reads it
+// and cannot change it, and the counts are no secret (novox/hq issue 454 — written 0600 before, which the
+// account could not read, so every night would have refused it and thinned nothing).
+func (b *Backups) retention(context.Context) (Retention, error) {
 	path := filepath.Join(b.Where.State, RetentionFile)
 	raw, err := os.ReadFile(path)
-	if errors.Is(err, os.ErrNotExist) {
-		return TodaysRetention, nil
-	}
-	if err != nil {
+	switch {
+	case errors.Is(err, fs.ErrNotExist):
+		return Retention{}, fmt.Errorf("%s %s", path, Missing)
+	case errors.Is(err, fs.ErrPermission):
+		return Retention{}, fmt.Errorf("%s %s (%v)", path, Unreadable, errors.Unwrap(err))
+	case err != nil:
 		return Retention{}, err
 	}
-	r, err := ParseRetention(string(raw))
+	keep, err := ParseRetention(string(raw))
 	if err != nil {
 		return Retention{}, fmt.Errorf("%s: %w", path, err)
 	}
-	return r, nil
+	return keep, nil
+}
+
+// InForce is the retention a night thins by, or why it thins nothing: what `backed-up` says beside each
+// module and what the health judges.
+type InForce struct {
+	*Retention
+	Refused string `json:"refused,omitempty"`
+	// Setting is the setting a refusal names, when one setting is what is wrong.
+	Setting string `json:"setting,omitempty"`
+	// Value is what that setting is now.
+	Value string `json:"-"`
+}
+
+func (b *Backups) inForce(ctx context.Context) InForce {
+	keep, err := b.retention(ctx)
+	if err == nil {
+		return InForce{Retention: &keep}
+	}
+	f := InForce{Refused: err.Error()}
+	var s *SettingRefused
+	if errors.As(err, &s) {
+		f.Setting, f.Value = s.Setting, s.Value
+	}
+	return f
 }

modules/restic/cmd/restic-backups/retention_test.go

@@ -21,8 +21,11 @@ import (
 func thinning(t *testing.T, retention *string) (string, []string) {
 	t.Helper()
 	where := placed(t, "# pg\npath /\n")
+	path := filepath.Join(where.State, RetentionFile)
 	if retention != nil {
-		must(t, os.WriteFile(filepath.Join(where.State, "retention.conf"), []byte(*retention), 0o600))
+		must(t, os.WriteFile(path, []byte(*retention), 0o600))
+	} else {
+		must(t, os.Remove(path))
 	}
 	forget := ""
 	run := func(_ context.Context, name string, args ...string) (string, error) {
@@ -53,11 +56,76 @@ func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) {
 	}
 }
 
-// A machine placed by a definition older than the file keeps today's rotation.
-func TestWithoutTheFileANightKeepsTodaysRetention(t *testing.T) {
-	got, _ := thinning(t, nil)
-	if want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6"; got != want {
-		t.Errorf("ran %q, want %q", got, want)
+// Every machine has had the file since mesh-catalog #200 rolled out (novox/hq issue 454), so a missing one is
+// a fault, never a reason to thin by 14/8/6: a lost file would otherwise thin by a number nobody set.
+func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) {
+	got, said := thinning(t, nil)
+	if got != "" {
+		t.Errorf("ran %q without a retention file", got)
+	}
+	all := strings.Join(said, "\n")
+	if !strings.Contains(all, "retention.conf is missing") || !strings.Contains(all, "every restore point is kept") {
+		t.Errorf("said %q, want it to say the file is missing and every restore point is kept", all)
+	}
+}
+
+// A file the account the night runs as may not read — the mode an older build of this module had the mesh
+// write it in (0600, root's) — thins nothing, and says why (novox/hq issue 454).
+func TestARetentionTheAccountCannotReadThinsNothingAndSaysWhy(t *testing.T) {
+	if os.Getuid() == 0 {
+		t.Skip("root reads a file of mode 0 itself")
+	}
+	where := placed(t, "# pg\npath /\n")
+	must(t, os.Chmod(filepath.Join(where.State, RetentionFile), 0))
+	forget := ""
+	var said []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		switch {
+		case name == "restic" && args[5] == "forget":
+			forget = "restic " + strings.Join(args[5:], " ")
+		case name == "restic" && args[5] == "backup":
+			return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
+		}
+		return "", nil
+	}
+	b := &Backups{Where: where, Run: run, Now: time.Now, Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
+	_, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	if forget != "" {
+		t.Errorf("thinned by a file it could not read: %q", forget)
+	}
+	if all := strings.Join(said, "\n"); !strings.Contains(all, Unreadable) || !strings.Contains(all, "every restore point is kept") {
+		t.Errorf("said %q", all)
+	}
+	h := b.Health(context.Background())
+	if h.State != "unhealthy" || !strings.Contains(h.Why, Unreadable) || !strings.Contains(h.Why, "0644") {
+		t.Errorf("%s: %q, want it unhealthy naming the cause", h.State, h.Why)
+	}
+}
+
+// The mesh writes the file readable, and root's: the account reads it and cannot change it.
+func TestTheManifestWritesTheRetentionReadable(t *testing.T) {
+	raw, err := os.ReadFile("../../module.json")
+	must(t, err)
+	var m struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	must(t, json.Unmarshal(raw, &m))
+	file, night := -1, -1
+	for i, r := range m.Resources {
+		switch r["id"] {
+		case "retention":
+			file = i
+			if r["mode"] != "0644" || r["owner"] != nil {
+				t.Errorf("the retention file is mode %v owner %v, want 0644 and root's", r["mode"], r["owner"])
+			}
+		case "night":
+			night = i
+		}
+	}
+	// Applied in declared order: the rollout that brings the night's process makes the file readable first.
+	if file < 0 || night < 0 || file > night {
+		t.Errorf("the retention file is declared at %d and the night at %d", file, night)
 	}
 }
 

modules/restic/go.mod

@@ -2,4 +2,4 @@ module restic
 
 go 1.25.0
 
-require git.novox.be/novox/mesh-sdk/go v0.1.7
+require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c

modules/restic/go.sum

@@ -1,2 +1,2 @@
-git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
-git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=

modules/restic/module.json

@@ -12,6 +12,11 @@
       ]
     }
   ],
+  "tools": [
+    "backup_health",
+    "backup_delete_retired",
+    "backup_deleted"
+  ],
   "own-secrets": {
     "repository": "${dir:state}/repository.secret"
   },
@@ -79,7 +84,7 @@
       "id": "retention",
       "type": "file",
       "path": "${dir:state}/retention.conf",
-      "mode": "0600",
+      "mode": "0644",
       "trusted": true,
       "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly and keep-monthly (novox/hq issue 416). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\n"
     },
@@ -92,6 +97,53 @@
       "id": "sqlite",
       "type": "package",
       "package": "sqlite"
+    },
+    {
+      "id": "night",
+      "type": "process",
+      "name": "restic-night",
+      "artifact": "tools",
+      "run": [
+        "./restic-backups",
+        "night"
+      ],
+      "user": "${machine:account}",
+      "env": {
+        "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
+        "MESH_BACKUP_REPOSITORY": "${dir:repository}",
+        "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
+        "MESH_BACKUP_STATE": "${dir:state}",
+        "MESH_BACKUP_DATA": "${dir:state}/data.conf"
+      },
+      "health": {
+        "kind": "tool",
+        "tool": "backup_health",
+        "interval": "60s",
+        "timeout": "20s",
+        "looks": 2,
+        "grace": "60s"
+      }
+    },
+    {
+      "id": "measure",
+      "type": "process",
+      "name": "restic-measure",
+      "artifact": "tools",
+      "run": [
+        "./restic-backups",
+        "measure"
+      ],
+      "user": "${machine:account}",
+      "env": {
+        "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
+        "MESH_BACKUP_REPOSITORY": "${dir:repository}",
+        "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
+        "MESH_BACKUP_STATE": "${dir:state}",
+        "MESH_BACKUP_DATA": "${dir:state}/data.conf"
+      },
+      "health": {
+        "kind": "unit"
+      }
     }
   ],
   "build": {