restic: raise a failed backup, a failed thinning and a refused retention as the module's health (novox/hq issue 454)

A failed night was a journal line only, so a typo in a retention setting
let the repository grow until a disk filled. The night and the measuring
now run as processes the node-engine judges: the night by backup_health,
which is unhealthy for a module not backed up, a night that did not run,
a failed thinning or a missing retention file, and waits naming the
setting when one is refused; the measuring by its unit. A file lock
keeps a night and the tool runtime's now and restore apart.

The retention file is root's alone, so the account reads it through sudo:
until now every night would have refused it as unreadable. Every machine
has the file, so a missing one is refused instead of thinned by 14/8/6.
backed-up says the retention in force and the last thinning per module.

modules/restic/cmd/restic-backups/backups.go

@@ -9,9 +9,9 @@
 //
 // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
 // restored on its own. Everything lands in one repository on the machine — deduplicated, so every
-// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
-// 8 weekly and 6 monthly unless the machine's settings say otherwise (retention.go). Against
-// mistakes, not disasters: nothing leaves the machine.
+// night is a complete restore point and only what changed costs space — and is thinned by the machine's
+// settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters:
+// nothing leaves the machine. How the last night and thinning went is the module's health (health.go).
 //
 // Root's: the dumps read every store and the repository holds every module's data, and the runtime
 // launching this binary runs as the operator's account, so restic and the run lines go through sudo
@@ -32,7 +32,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
-	"sync"
+	"syscall"
 	"time"
 )
 
@@ -178,13 +178,44 @@ func whereFromEnv() (Where, error) {
 }
 
 // Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never
-// share a dump.
+// share a dump — across the night's process and the tool runtime alike (hold).
 type Backups struct {
 	Where Where
 	Run   Runner
 	Now   func() time.Time
 	Say   func(format string, args ...any)
-	mu    sync.Mutex
+	read  retentionRead
+}
+
+// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a
+// retired item's deletion in the tool runtime (novox/hq issue 454), so a mutex in one of them no longer
+// keeps the other out.
+const (
+	nightLock   = "night.lock"
+	measureLock = "measure.lock"
+)
+
+// hold takes one of the locks, waiting for whoever has it, and answers how to let go. A lock is the open
+// file's, so two holders in one process exclude each other as two processes do.
+func (b *Backups) hold(name string) (func(), error) {
+	f, err := os.OpenFile(filepath.Join(b.Where.State, name), os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("the lock %s cannot be opened: %w", name, err)
+	}
+	for {
+		err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX)
+		if !errors.Is(err, syscall.EINTR) {
+			break
+		}
+	}
+	if err != nil {
+		f.Close()
+		return nil, fmt.Errorf("the lock %s cannot be taken: %w", name, err)
+	}
+	return func() {
+		_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
 }
 
 // exists is whether a path is there, asked as root: a store's directory is often its own user's
@@ -299,16 +330,27 @@ func snapshotOf(out string) string {
 	return id
 }
 
-// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome.
+// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and
+// keeps how the night as a whole and its thinning went for the health (novox/hq issue 454).
 func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if err := b.ensureRepository(ctx); err != nil {
+	release, err := b.hold(nightLock)
+	if err != nil {
 		return nil, err
 	}
+	defer release()
+	run := b.LastRun()
+	run.At = b.Now().UTC()
+	failed := func(err error) (map[string]Night, error) {
+		run.Failed = err.Error()
+		b.keepRun(run)
+		return nil, err
+	}
+	if err := b.ensureRepository(ctx); err != nil {
+		return failed(err)
+	}
 	all, err := b.Declared()
 	if err != nil {
-		return nil, err
+		return failed(err)
 	}
 	chosen := all
 	if only != "" {
@@ -321,6 +363,7 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er
 			}
 		}
 		if len(chosen) == 0 {
+			// Asked of one module that is not here: nothing ran, and the last night stands as it was.
 			return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names))
 		}
 	}
@@ -331,16 +374,59 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er
 	}
 	// A retention that does not read thins nothing: keeping too much costs space, thinning by a value
 	// nobody meant loses restore points (novox/hq issue 416).
-	keep, err := b.retention()
+	thinned := Thinning{At: b.Now().UTC()}
+	keep, err := b.retention(ctx)
 	if err != nil {
+		thinned.Refused = err.Error()
 		b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err)
 	} else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
 		"--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil {
+		thinned.Kept, thinned.Error = &keep, err.Error()
 		b.Say("thinning the restore points failed, and every one is kept: %v", err)
+	} else {
+		thinned.Kept, thinned.OK = &keep, true
 	}
+	run.Failed, run.Thinned = "", &thinned
+	b.keepRun(run)
 	return outcome, nil
 }
 
+// Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its
+// thinning. Each module's own outcome is its Night.
+type Run struct {
+	At time.Time `json:"at"`
+	// Failed is why it did not run at all: the repository would not open, or what to back up does not read.
+	Failed  string    `json:"failed,omitempty"`
+	Thinned *Thinning `json:"thinned,omitempty"`
+}
+
+// Thinning is how the last thinning went: the retention it thinned by, or why it thinned nothing.
+type Thinning struct {
+	At      time.Time  `json:"at"`
+	OK      bool       `json:"ok"`
+	Kept    *Retention `json:"kept,omitempty"`
+	Refused string     `json:"refused,omitempty"`
+	Error   string     `json:"error,omitempty"`
+}
+
+func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") }
+
+// LastRun is how the last night went as a whole; empty before the first.
+func (b *Backups) LastRun() Run {
+	var r Run
+	if raw, err := os.ReadFile(b.runFile()); err == nil {
+		_ = json.Unmarshal(raw, &r)
+	}
+	return r
+}
+
+func (b *Backups) keepRun(r Run) {
+	raw, _ := json.MarshalIndent(r, "", "  ")
+	if err := os.WriteFile(b.runFile(), append(raw, '\n'), 0o600); err != nil {
+		b.Say("recording how the night went failed: %v", err)
+	}
+}
+
 func orNothing(names []string) string {
 	if len(names) == 0 {
 		return "nothing"
@@ -376,6 +462,11 @@ type ModuleBackups struct {
 	LastNight     *Night    `json:"lastNight"`
 	RestorePoints int       `json:"restorePoints"`
 	Newest        *Snapshot `json:"newest,omitempty"`
+	// Retention is how many of its restore points the next thinning keeps, or why it thins nothing; Thinned
+	// how the last thinning went (novox/hq issue 454). Said beside each module, which restic thins on its
+	// own, so the answer stays the list the controller reads.
+	Retention InForce   `json:"retention"`
+	Thinned   *Thinning `json:"thinned,omitempty"`
 	// Data is every item the module declares, measured, with its newest good backup (novox/hq ADR
 	// 0233): what the controller's self-check reads.
 	Data []ItemReport `json:"data"`
@@ -396,6 +487,8 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups,
 	good := b.Good()
 	measured := b.Measurements()
 	snaps, _ := b.Snapshots(ctx, module)
+	inForce := b.inForce(ctx)
+	thinned := b.LastRun().Thinned
 	out := []ModuleBackups{}
 	listed := map[string]bool{}
 	for _, d := range declared {
@@ -403,7 +496,7 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups,
 			continue
 		}
 		listed[d.Module] = true
-		m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths,
+		m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned,
 			Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])}
 		if n, ok := nights[d.Module]; ok {
 			m.LastNight = &n
@@ -440,8 +533,11 @@ type Restored struct {
 // state as restored/<stamp>/<module>/<path> (besideIsUnsafe). A target that already exists is refused, never
 // overwritten.
 func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
+	release, err := b.hold(nightLock)
+	if err != nil {
+		return nil, err
+	}
+	defer release()
 	mine, err := b.Snapshots(ctx, module)
 	if err != nil {
 		return nil, err
@@ -583,9 +679,12 @@ func (b *Backups) isFile(ctx context.Context, id, path string) (bool, error) {
 
 // Check is the weekly look at the repository's own integrity, with a sample of the data read back.
 func (b *Backups) Check(ctx context.Context) error {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	_, err := b.restic(ctx, "check", "--read-data-subset", "5%")
+	release, err := b.hold(nightLock)
+	if err != nil {
+		return err
+	}
+	defer release()
+	_, err = b.restic(ctx, "check", "--read-data-subset", "5%")
 	return err
 }