jschoubben
3cce15fa81
mesh/merge-gate head 3cce15fa811e merged into trunk 835bf22b7d40 — pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with …
mesh/repo-check head 3cce15fa811e merged into trunk 835bf22b7d40 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/check failed: unit: modules/gitea.test: [gitea] where the poll of merges last looked from could not be read, so none is kept — no runtime | 3cce1…
A restic outside the night's lock, such as a listing while a delivery plan backs up data modules before their moves, held the lock 246 ms before the thinning asked for it, and the thinning failed at once, raising an urgent condition for a timing collision. restic's --retry-lock now waits up to ten minutes on both of the thinning's forgets; past that it fails with restic's words as before.
208 lines
8.0 KiB
Go
208 lines
8.0 KiB
Go
package main
|
|
|
|
// The retention is a preference (novox/hq issue 416, ADR 0262): the mesh writes the keep-daily,
|
|
// keep-weekly and keep-monthly settings into a file in this module's state, and a night thins the
|
|
// restore points by what that file says. A value that is not a count thins nothing.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// thinning runs one night over a fake restic and answers the forget line it ran ("" when none) and
|
|
// what the module said.
|
|
func thinning(t *testing.T, retention *string) (string, []string) {
|
|
t.Helper()
|
|
where := placed(t, "# pg\npath /\n")
|
|
path := filepath.Join(where.State, RetentionFile)
|
|
if retention != nil {
|
|
must(t, os.WriteFile(path, []byte(*retention), 0o600))
|
|
} else {
|
|
must(t, os.Remove(path))
|
|
}
|
|
forget := ""
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
if name == "restic" && args[5] == "forget" {
|
|
forget = "restic " + strings.Join(args[5:], " ")
|
|
}
|
|
if name == "restic" && args[5] == "backup" {
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
var said []string
|
|
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: say}
|
|
outcome, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if !outcome["pg"].OK {
|
|
t.Fatalf("the night itself failed: %+v", outcome["pg"])
|
|
}
|
|
return forget, said
|
|
}
|
|
|
|
func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) {
|
|
conf := "# Written by the mesh.\ndaily=7\nweekly=4\nmonthly=24\nhand=10\n"
|
|
got, _ := thinning(t, &conf)
|
|
if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24 --keep-tag hand --retry-lock 10m0s"; got != want {
|
|
t.Errorf("ran %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// Every machine has had the file since mesh-catalog #200 rolled out (novox/hq issue 454), so a missing one is
|
|
// a fault, never a reason to thin by 14/8/6: a lost file would otherwise thin by a number nobody set.
|
|
func TestWithoutTheFileANightThinsNothingAndSaysTheFileIsMissing(t *testing.T) {
|
|
got, said := thinning(t, nil)
|
|
if got != "" {
|
|
t.Errorf("ran %q without a retention file", got)
|
|
}
|
|
all := strings.Join(said, "\n")
|
|
if !strings.Contains(all, "retention.conf is missing") || !strings.Contains(all, "every restore point is kept") {
|
|
t.Errorf("said %q, want it to say the file is missing and every restore point is kept", all)
|
|
}
|
|
}
|
|
|
|
// A file the account the night runs as may not read — the mode an older build of this module had the mesh
|
|
// write it in (0600, root's) — thins nothing, and says why (novox/hq issue 454).
|
|
func TestARetentionTheAccountCannotReadThinsNothingAndSaysWhy(t *testing.T) {
|
|
if os.Getuid() == 0 {
|
|
t.Skip("root reads a file of mode 0 itself")
|
|
}
|
|
where := placed(t, "# pg\npath /\n")
|
|
must(t, os.Chmod(filepath.Join(where.State, RetentionFile), 0))
|
|
forget := ""
|
|
var said []string
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
switch {
|
|
case name == "restic" && args[5] == "forget":
|
|
forget = "restic " + strings.Join(args[5:], " ")
|
|
case name == "restic" && args[5] == "backup":
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: where, Run: run, Now: time.Now, Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if forget != "" {
|
|
t.Errorf("thinned by a file it could not read: %q", forget)
|
|
}
|
|
if all := strings.Join(said, "\n"); !strings.Contains(all, Unreadable) || !strings.Contains(all, "every restore point is kept") {
|
|
t.Errorf("said %q", all)
|
|
}
|
|
h := b.Health(context.Background())
|
|
if h.State != "unhealthy" || !strings.Contains(h.Why, Unreadable) || !strings.Contains(h.Why, "0644") {
|
|
t.Errorf("%s: %q, want it unhealthy naming the cause", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// The mesh writes the file readable, and root's: the account reads it and cannot change it.
|
|
func TestTheManifestWritesTheRetentionReadable(t *testing.T) {
|
|
raw, err := os.ReadFile("../../module.json")
|
|
must(t, err)
|
|
var m struct {
|
|
Resources []map[string]any `json:"resources"`
|
|
}
|
|
must(t, json.Unmarshal(raw, &m))
|
|
file, night := -1, -1
|
|
for i, r := range m.Resources {
|
|
switch r["id"] {
|
|
case "retention":
|
|
file = i
|
|
if r["mode"] != "0644" || r["owner"] != nil {
|
|
t.Errorf("the retention file is mode %v owner %v, want 0644 and root's", r["mode"], r["owner"])
|
|
}
|
|
case "night":
|
|
night = i
|
|
}
|
|
}
|
|
// Applied in declared order: the rollout that brings the night's process makes the file readable first.
|
|
if file < 0 || night < 0 || file > night {
|
|
t.Errorf("the retention file is declared at %d and the night at %d", file, night)
|
|
}
|
|
}
|
|
|
|
// A value that is not a count from 1 to 9999 is never handed to restic: nothing is thinned, every
|
|
// restore point is kept, and the module says which setting is wrong (issue 346: a raw setting is an
|
|
// injection point).
|
|
func TestARetentionThatIsNotACountThinsNothingAndSaysWhich(t *testing.T) {
|
|
for _, c := range []struct{ conf, names string }{
|
|
{"daily=0\nweekly=8\nmonthly=6\n", "keep-daily"},
|
|
{"daily=14\nweekly=-1\nmonthly=6\n", "keep-weekly"},
|
|
{"daily=14\nweekly=8\nmonthly=6 --keep-last 1\n", "keep-monthly"},
|
|
{"daily=14\nweekly=8\nmonthly=10000\n", "keep-monthly"},
|
|
{"daily=fourteen\nweekly=8\nmonthly=6\n", "keep-daily"},
|
|
{"daily=\nweekly=8\nmonthly=6\n", "keep-daily"},
|
|
{"daily=014\nweekly=8\nmonthly=6\n", "keep-daily"},
|
|
{"weekly=8\nmonthly=6\n", "keep-daily"},
|
|
{"daily=14\nweekly=8\nmonthly=6\nlast=1\n", "last"},
|
|
{"daily=14\ndaily=3\nweekly=8\nmonthly=6\n", "daily"},
|
|
{"daily 14\nweekly=8\nmonthly=6\n", "key=value"},
|
|
{"daily=14\nweekly=8\nmonthly=6\nhand=0\n", "keep-hand-taken-days"},
|
|
{"daily=14\nweekly=8\nmonthly=6\n", "hand= line is missing"},
|
|
} {
|
|
got, said := thinning(t, &c.conf)
|
|
if got != "" {
|
|
t.Errorf("%q: ran %q", c.conf, got)
|
|
}
|
|
all := strings.Join(said, "\n")
|
|
if !strings.Contains(all, c.names) || !strings.Contains(all, "every restore point is kept") {
|
|
t.Errorf("%q: said %q, want it to name %s and say every restore point is kept", c.conf, all, c.names)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The manifest declares the retention's config values with today's values as their defaults, and its file,
|
|
// filled with those defaults, reads as today's retention: nothing changes on a machine that sets none.
|
|
func TestTheManifestsDefaultsAreTodaysRetention(t *testing.T) {
|
|
raw, err := os.ReadFile("../../module.json")
|
|
must(t, err)
|
|
var m struct {
|
|
Config map[string]struct {
|
|
Type string `json:"type"`
|
|
Description string `json:"description"`
|
|
Minimum *float64 `json:"minimum"`
|
|
Maximum *float64 `json:"maximum"`
|
|
Default any `json:"default"`
|
|
} `json:"config"`
|
|
Resources []map[string]any `json:"resources"`
|
|
}
|
|
must(t, json.Unmarshal(raw, &m))
|
|
want := map[string]float64{"keep-daily": 14, "keep-weekly": 8, "keep-monthly": 6, "keep-hand-taken-days": 7}
|
|
for k, v := range want {
|
|
s, ok := m.Config[k]
|
|
if !ok || s.Type != "number" || s.Default != v || s.Description == "" || s.Minimum == nil || *s.Minimum != 1 ||
|
|
s.Maximum == nil || *s.Maximum != 9999 {
|
|
t.Errorf("config value %s: %+v, want a number from 1 to 9999 defaulting to %v with a description", k, s, v)
|
|
}
|
|
}
|
|
content := ""
|
|
for _, r := range m.Resources {
|
|
if r["id"] == "retention" {
|
|
if r["path"] != "${dir:state}/"+RetentionFile {
|
|
t.Errorf("the retention file is placed at %v, where this program does not read it", r["path"])
|
|
}
|
|
content = templateText(t, r)
|
|
}
|
|
}
|
|
if content == "" {
|
|
t.Fatal("no retention resource in the manifest")
|
|
}
|
|
filled := regexp.MustCompile(`\$\{config:([a-z-]+)\}`).ReplaceAllStringFunc(content, func(p string) string {
|
|
k := p[len("${config:") : len(p)-1]
|
|
return fmt.Sprint(m.Config[k].Default)
|
|
})
|
|
got, err := ParseRetention(filled)
|
|
must(t, err)
|
|
if got != TodaysRetention {
|
|
t.Errorf("the defaults read as %+v, today is %+v", got, TodaysRetention)
|
|
}
|
|
}
|