restic: take the retention from the machine's settings, so a disk can keep more or fewer restore points without a new build (novox/hq issue 416)

The keep-daily, keep-weekly and keep-monthly preferences default to
today's 14, 8 and 6. The night reads them from a file in its state and
refuses anything but a count from 1 to 9999: it then thins nothing and
says which setting is wrong, never handing a raw value to restic.

modules/restic/cmd/restic-backups/backups.go

@@ -10,7 +10,7 @@
 // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
 // restored on its own. Everything lands in one repository on the machine — deduplicated, so every
 // night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
-// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
+// 8 weekly and 6 monthly unless the machine's settings say otherwise (retention.go). Against mistakes, not disasters: nothing leaves the machine.
 //
 // Root's: the dumps read every store and the repository holds every module's data, and the runtime
 // launching this binary runs as the operator's account, so restic and the run lines go through sudo
@@ -144,9 +144,6 @@ type Night struct {
 	Error    string    `json:"error,omitempty"`
 }
 
-// Keep is the rotation (novox/hq ADR 0214).
-var Keep = struct{ Daily, Weekly, Monthly int }{14, 8, 6}
-
 func tagOf(module string) string { return "module=" + module }
 
 // Where is where the mesh put this module's things.
@@ -331,8 +328,13 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er
 		outcome[d.Module] = b.one(ctx, d)
 		b.record(d.Module, outcome[d.Module])
 	}
-	if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
-		"--keep-daily", fmt.Sprint(Keep.Daily), "--keep-weekly", fmt.Sprint(Keep.Weekly), "--keep-monthly", fmt.Sprint(Keep.Monthly)); err != nil {
+	// A retention that does not read thins nothing: keeping too much costs space, thinning by a value
+	// nobody meant loses restore points (novox/hq issue 416).
+	keep, err := b.retention()
+	if err != nil {
+		b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err)
+	} else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
+		"--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil {
 		b.Say("thinning the restore points failed, and every one is kept: %v", err)
 	}
 	return outcome, nil

modules/restic/cmd/restic-backups/retention.go

@@ -0,0 +1,83 @@
+package main
+
+// The retention as a preference (novox/hq issue 416, ADR 0262). The mesh writes the settings
+// keep-daily, keep-weekly and keep-monthly into RetentionFile in this module's state, each defaulting
+// to today's rotation, so a machine that sets none thins exactly as before. The controller does not
+// type a setting yet (ADR 0164 is proposed), so this program is where a bad value is caught: it is
+// refused here and never handed to restic (issue 346: a raw setting is an injection point).
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Retention is how many restore points each period keeps.
+type Retention struct{ Daily, Weekly, Monthly int }
+
+// TodaysRetention is the rotation before it was a preference (novox/hq ADR 0214), and still the
+// manifest's default.
+var TodaysRetention = Retention{14, 8, 6}
+
+// RetentionFile is the file the mesh writes the retention into, in this module's state.
+const RetentionFile = "retention.conf"
+
+// A count from 1 to 9999 with no leading zero: 0 would make restic ignore the period, and a number
+// past four digits is a typo, not a policy.
+var count = regexp.MustCompile(`^[1-9][0-9]{0,3}$`)
+
+// ParseRetention reads daily=, weekly= and monthly=, each once and each a count; anything else is
+// refused naming the setting it came from.
+func ParseRetention(text string) (Retention, error) {
+	settingOf := map[string]string{"daily": "keep-daily", "weekly": "keep-weekly", "monthly": "keep-monthly"}
+	seen := map[string]int{}
+	for _, l := range strings.Split(text, "\n") {
+		l = strings.TrimSpace(l)
+		if l == "" || strings.HasPrefix(l, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(l, "=")
+		if !ok {
+			return Retention{}, fmt.Errorf("%q is not key=value", l)
+		}
+		k, v = strings.TrimSpace(k), strings.TrimSpace(v)
+		setting, known := settingOf[k]
+		switch {
+		case !known:
+			return Retention{}, fmt.Errorf("%q is no retention this module reads (daily, weekly, monthly)", k)
+		case seen[k] != 0:
+			return Retention{}, fmt.Errorf("%s is given twice", k)
+		case !count.MatchString(v):
+			return Retention{}, fmt.Errorf("the %s setting is %q; it is a count from 1 to 9999", setting, v)
+		}
+		seen[k], _ = strconv.Atoi(v)
+	}
+	for _, k := range []string{"daily", "weekly", "monthly"} {
+		if seen[k] == 0 {
+			return Retention{}, fmt.Errorf("the %s setting is missing", settingOf[k])
+		}
+	}
+	return Retention{seen["daily"], seen["weekly"], seen["monthly"]}, nil
+}
+
+// retention is this machine's: today's where the module was placed by a definition older than the
+// file, else what the file says.
+func (b *Backups) retention() (Retention, error) {
+	path := filepath.Join(b.Where.State, RetentionFile)
+	raw, err := os.ReadFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return TodaysRetention, nil
+	}
+	if err != nil {
+		return Retention{}, err
+	}
+	r, err := ParseRetention(string(raw))
+	if err != nil {
+		return Retention{}, fmt.Errorf("%s: %w", path, err)
+	}
+	return r, nil
+}

modules/restic/cmd/restic-backups/retention_test.go

@@ -0,0 +1,134 @@
+package main
+
+// The retention is a preference (novox/hq issue 416, ADR 0262): the mesh writes the keep-daily,
+// keep-weekly and keep-monthly settings into a file in this module's state, and a night thins the
+// restore points by what that file says. A value that is not a count thins nothing.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+)
+
+// thinning runs one night over a fake restic and answers the forget line it ran ("" when none) and
+// what the module said.
+func thinning(t *testing.T, retention *string) (string, []string) {
+	t.Helper()
+	where := placed(t, "# pg\npath /\n")
+	if retention != nil {
+		must(t, os.WriteFile(filepath.Join(where.State, "retention.conf"), []byte(*retention), 0o600))
+	}
+	forget := ""
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "restic" && args[5] == "forget" {
+			forget = "restic " + strings.Join(args[5:], " ")
+		}
+		if name == "restic" && args[5] == "backup" {
+			return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
+		}
+		return "", nil
+	}
+	var said []string
+	say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
+	b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: say}
+	outcome, err := b.BackUp(context.Background(), "")
+	must(t, err)
+	if !outcome["pg"].OK {
+		t.Fatalf("the night itself failed: %+v", outcome["pg"])
+	}
+	return forget, said
+}
+
+func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) {
+	conf := "# Written by the mesh.\ndaily=7\nweekly=4\nmonthly=24\n"
+	got, _ := thinning(t, &conf)
+	if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; got != want {
+		t.Errorf("ran %q, want %q", got, want)
+	}
+}
+
+// A machine placed by a definition older than the file keeps today's rotation.
+func TestWithoutTheFileANightKeepsTodaysRetention(t *testing.T) {
+	got, _ := thinning(t, nil)
+	if want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6"; got != want {
+		t.Errorf("ran %q, want %q", got, want)
+	}
+}
+
+// A value that is not a count from 1 to 9999 is never handed to restic: nothing is thinned, every
+// restore point is kept, and the module says which setting is wrong (issue 346: a raw setting is an
+// injection point).
+func TestARetentionThatIsNotACountThinsNothingAndSaysWhich(t *testing.T) {
+	for _, c := range []struct{ conf, names string }{
+		{"daily=0\nweekly=8\nmonthly=6\n", "keep-daily"},
+		{"daily=14\nweekly=-1\nmonthly=6\n", "keep-weekly"},
+		{"daily=14\nweekly=8\nmonthly=6 --keep-last 1\n", "keep-monthly"},
+		{"daily=14\nweekly=8\nmonthly=10000\n", "keep-monthly"},
+		{"daily=fourteen\nweekly=8\nmonthly=6\n", "keep-daily"},
+		{"daily=\nweekly=8\nmonthly=6\n", "keep-daily"},
+		{"daily=014\nweekly=8\nmonthly=6\n", "keep-daily"},
+		{"weekly=8\nmonthly=6\n", "keep-daily"},
+		{"daily=14\nweekly=8\nmonthly=6\nlast=1\n", "last"},
+		{"daily=14\ndaily=3\nweekly=8\nmonthly=6\n", "daily"},
+		{"daily 14\nweekly=8\nmonthly=6\n", "key=value"},
+	} {
+		got, said := thinning(t, &c.conf)
+		if got != "" {
+			t.Errorf("%q: ran %q", c.conf, got)
+		}
+		all := strings.Join(said, "\n")
+		if !strings.Contains(all, c.names) || !strings.Contains(all, "every restore point is kept") {
+			t.Errorf("%q: said %q, want it to name %s and say every restore point is kept", c.conf, all, c.names)
+		}
+	}
+}
+
+// The manifest declares the three preferences with today's values as their defaults, and its file,
+// filled with those defaults, reads as today's retention: nothing changes on a machine that sets none.
+func TestTheManifestsDefaultsAreTodaysRetention(t *testing.T) {
+	raw, err := os.ReadFile("../../module.json")
+	must(t, err)
+	var m struct {
+		Settings map[string]struct {
+			Kind    string `json:"kind"`
+			Default any    `json:"default"`
+			Why     string `json:"why"`
+		} `json:"settings"`
+		Resources []map[string]any `json:"resources"`
+	}
+	must(t, json.Unmarshal(raw, &m))
+	want := map[string]float64{"keep-daily": 14, "keep-weekly": 8, "keep-monthly": 6}
+	for k, v := range want {
+		s, ok := m.Settings[k]
+		if !ok || s.Kind != "preference" || s.Default != v || s.Why == "" {
+			t.Errorf("setting %s: %+v, want a preference defaulting to %v with why", k, s, v)
+		}
+	}
+	content := ""
+	for _, r := range m.Resources {
+		if r["id"] == "retention" {
+			if r["path"] != "${dir:state}/"+RetentionFile {
+				t.Errorf("the retention file is placed at %v, where this program does not read it", r["path"])
+			}
+			content, _ = r["content"].(string)
+		}
+	}
+	if content == "" {
+		t.Fatal("no retention resource in the manifest")
+	}
+	filled := regexp.MustCompile(`\$\{setting:([a-z-]+)\}`).ReplaceAllStringFunc(content, func(p string) string {
+		k := p[len("${setting:") : len(p)-1]
+		return fmt.Sprint(m.Settings[k].Default)
+	})
+	got, err := ParseRetention(filled)
+	must(t, err)
+	if got != TodaysRetention {
+		t.Errorf("the defaults read as %+v, today is %+v", got, TodaysRetention)
+	}
+}

modules/restic/module.json

@@ -32,6 +32,23 @@
       }
     ]
   },
+  "settings": {
+    "keep-daily": {
+      "kind": "preference",
+      "default": 14,
+      "why": "two weeks of nightly restore points undo a mistake noticed within a fortnight; a count from 1 to 9999, read by the night, which thins nothing while it does not read (novox/hq issue 416)"
+    },
+    "keep-weekly": {
+      "kind": "preference",
+      "default": 8,
+      "why": "two months of weekly restore points behind the dailies; a count from 1 to 9999, refused by the night like keep-daily"
+    },
+    "keep-monthly": {
+      "kind": "preference",
+      "default": 6,
+      "why": "half a year of monthly restore points behind the weeklies; a count from 1 to 9999, refused by the night like keep-daily"
+    }
+  },
   "resources": [
     {
       "id": "state",
@@ -58,6 +75,14 @@
       "mode": "0600",
       "content": "# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.\n${contribution:node-backup:data}"
     },
+    {
+      "id": "retention",
+      "type": "file",
+      "path": "${dir:state}/retention.conf",
+      "mode": "0600",
+      "trusted": true,
+      "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly and keep-monthly (novox/hq issue 416). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\n"
+    },
     {
       "id": "tool",
       "type": "package",