restic: raise a failed backup, a failed thinning and a refused retention as the module's health (novox/hq issue 454)

A failed night was a journal line only, so a typo in a retention setting
let the repository grow until a disk filled. The night and the measuring
now run as processes the node-engine judges: the night by backup_health,
which is unhealthy for a module not backed up, a night that did not run,
a failed thinning or a missing retention file, and waits naming the
setting when one is refused; the measuring by its unit. A file lock
keeps a night and the tool runtime's now and restore apart.

The retention file is root's alone, so the account reads it through sudo:
until now every night would have refused it as unreadable. Every machine
has the file, so a missing one is refused instead of thinned by 14/8/6.
backed-up says the retention in force and the last thinning per module.

modules/restic/cmd/restic-backups/data.go

@@ -157,8 +157,6 @@ func (b *Backups) Measurements() map[string]map[string]Measured {
 	return out
 }
 
-var measuring sync.Mutex
-
 // The ways an item is measured.
 const (
 	measureWalk    = "walk"
@@ -364,8 +362,12 @@ func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
 // measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the
 // hourly round reads counters, and a walk runs at most once a day.
 func (b *Backups) MeasureAll(ctx context.Context, walks bool) error {
-	measuring.Lock()
-	defer measuring.Unlock()
+	// The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454).
+	release, err := b.hold(measureLock)
+	if err != nil {
+		return err
+	}
+	defer release()
 	items, err := b.Items()
 	if err != nil {
 		return err
@@ -552,8 +554,13 @@ func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm
 }
 
 func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
+	release, err := b.hold(nightLock)
+	if err != nil {
+		d.Running, d.Done, d.Error = false, true, err.Error()
+		b.keepDeletion(path, d)
+		return
+	}
+	defer release()
 	finish := func(err error) {
 		d.Running, d.Done = false, true
 		if err != nil {