restic: raise a night that never happens, and have the mesh write the retention readable (novox/hq issue 454)

A hung night, a lock held for hours or a lock that cannot be taken left no
failed night behind, so it stayed healthy. The night's process now counts
how long the machine was up since its newest good night, by the monotonic
clock so a sleeping laptop is not blamed, and past 26 hours it is raised.

The retention file is written 0644 and root's instead of read through
sudo: the account reads it and still cannot change it. An unreadable file
names its cause, and the healthy answer says only what was seen.

modules/restic/cmd/restic-backups/retention_test.go

@@ -69,23 +69,18 @@ func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) {
 	}
 }
 
-// The mesh writes the file as root's alone (a trusted file, novox/hq issue 339), and the night runs as the
-// operator account: what the account cannot read is read as root, as every other root's file here is
-// (novox/hq issue 454). Before, every machine's night refused it as unreadable and thinned nothing.
-func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) {
-	where := placed(t, "# pg\npath /\n")
-	path := filepath.Join(where.State, RetentionFile)
-	must(t, os.WriteFile(path, []byte("daily=7\nweekly=4\nmonthly=24\n"), 0o600))
-	must(t, os.Chmod(path, 0))
+// A file the account the night runs as may not read — the mode an older build of this module had the mesh
+// write it in (0600, root's) — thins nothing, and says why (novox/hq issue 454).
+func TestARetentionTheAccountCannotReadThinsNothingAndSaysWhy(t *testing.T) {
 	if os.Getuid() == 0 {
-		t.Skip("root reads a file of mode 0 itself; nothing to escalate")
+		t.Skip("root reads a file of mode 0 itself")
 	}
-	forget, catted := "", ""
+	where := placed(t, "# pg\npath /\n")
+	must(t, os.Chmod(filepath.Join(where.State, RetentionFile), 0))
+	forget := ""
+	var said []string
 	run := func(_ context.Context, name string, args ...string) (string, error) {
 		switch {
-		case name == "cat":
-			catted = strings.Join(args, " ")
-			return "daily=7\nweekly=4\nmonthly=24\n", nil
 		case name == "restic" && args[5] == "forget":
 			forget = "restic " + strings.Join(args[5:], " ")
 		case name == "restic" && args[5] == "backup":
@@ -93,14 +88,44 @@ func TestARetentionOnlyRootMayReadIsReadAsRoot(t *testing.T) {
 		}
 		return "", nil
 	}
-	b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet}
+	b := &Backups{Where: where, Run: run, Now: time.Now, Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
 	_, err := b.BackUp(context.Background(), "")
 	must(t, err)
-	if catted != "-- "+path {
-		t.Errorf("read %q as root, want %q", catted, "-- "+path)
+	if forget != "" {
+		t.Errorf("thinned by a file it could not read: %q", forget)
 	}
-	if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; forget != want {
-		t.Errorf("ran %q, want %q", forget, want)
+	if all := strings.Join(said, "\n"); !strings.Contains(all, Unreadable) || !strings.Contains(all, "every restore point is kept") {
+		t.Errorf("said %q", all)
+	}
+	h := b.Health(context.Background())
+	if h.State != "unhealthy" || !strings.Contains(h.Why, Unreadable) || !strings.Contains(h.Why, "0644") {
+		t.Errorf("%s: %q, want it unhealthy naming the cause", h.State, h.Why)
+	}
+}
+
+// The mesh writes the file readable, and root's: the account reads it and cannot change it.
+func TestTheManifestWritesTheRetentionReadable(t *testing.T) {
+	raw, err := os.ReadFile("../../module.json")
+	must(t, err)
+	var m struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	must(t, json.Unmarshal(raw, &m))
+	file, night := -1, -1
+	for i, r := range m.Resources {
+		switch r["id"] {
+		case "retention":
+			file = i
+			if r["mode"] != "0644" || r["owner"] != nil {
+				t.Errorf("the retention file is mode %v owner %v, want 0644 and root's", r["mode"], r["owner"])
+			}
+		case "night":
+			night = i
+		}
+	}
+	// Applied in declared order: the rollout that brings the night's process makes the file readable first.
+	if file < 0 || night < 0 || file > night {
+		t.Errorf("the retention file is declared at %d and the night at %d", file, night)
 	}
 }