package main // The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and // a refused retention each raise a condition naming the module and what to do, instead of a journal line // nobody reads until a disk fills. import ( "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/health" ) // nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails // with thinning when it is not empty. type nightly struct { failing map[string]bool thinning string } func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) { switch { case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]: return "", errors.New("the dump failed: no space left on device") case name == "restic" && args[5] == "backup": return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil case name == "restic" && args[5] == "forget" && n.thinning != "": return "", errors.New(n.thinning) } return "", nil } func nightAt(t *testing.T) (*Backups, *nightly) { t.Helper() n := &nightly{failing: map[string]bool{}} b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run, Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet} return b, n } func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) { b, n := nightAt(t) n.failing["mail"] = true _, err := b.BackUp(context.Background(), "") must(t, err) h := b.Health(context.Background()) if h.State != health.StateUnhealthy { t.Fatalf("a failed backup is %s: %s", h.State, h.Why) } for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} { if !strings.Contains(h.Why, want) { t.Errorf("said %q, want it to hold %q", h.Why, want) } } if strings.Contains(h.Why, "pg was NOT") { t.Errorf("pg was backed up and is said as failed: %q", h.Why) } // Backed up again, it clears. n.failing["mail"] = false _, err = b.BackUp(context.Background(), "mail") must(t, err) if h := b.Health(context.Background()); h.State != health.StateHealthy { t.Fatalf("backed up again, it is %s: %s", h.State, h.Why) } } func TestANightThatDidNotRunIsUnhealthy(t *testing.T) { b, _ := nightAt(t) b.Run = func(_ context.Context, name string, args ...string) (string, error) { return "", errors.New("Fatal: wrong password or no key found") } if _, err := b.BackUp(context.Background(), ""); err == nil { t.Fatal("a repository that will not open ran a night") } h := b.Health(context.Background()) if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") { t.Fatalf("a night that did not run: %s: %q", h.State, h.Why) } } func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) { b, n := nightAt(t) n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively" _, err := b.BackUp(context.Background(), "") must(t, err) h := b.Health(context.Background()) if h.State != health.StateUnhealthy { t.Fatalf("a failed thinning is %s: %s", h.State, h.Why) } for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} { if !strings.Contains(h.Why, want) { t.Errorf("said %q, want it to hold %q", h.Why, want) } } n.thinning = "" _, err = b.BackUp(context.Background(), "") must(t, err) if h := b.Health(context.Background()); h.State != health.StateHealthy { t.Fatalf("thinned again, it is %s: %s", h.State, h.Why) } } // A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283), // and the controller says it as the module's needs-operator condition with the act. func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) { b, _ := nightAt(t) must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600)) h := b.Health(context.Background()) if h.State != health.StateWaits { t.Fatalf("a refused setting is %s: %s", h.State, h.Why) } if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil { t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits) } for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} { if !strings.Contains(h.Why, want) { t.Errorf("said %q, want it to hold %q", h.Why, want) } } } // A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the // operator gave: it is a fault, and says to push the machine. func TestAMissingRetentionFileIsUnhealthy(t *testing.T) { b, _ := nightAt(t) must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile))) h := b.Health(context.Background()) if h.State != health.StateUnhealthy { t.Fatalf("a missing retention file is %s: %s", h.State, h.Why) } for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} { if !strings.Contains(h.Why, want) { t.Errorf("said %q, want it to hold %q", h.Why, want) } } } // A refused setting beside a failed backup: unhealthy, still naming the setting it waits for. func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) { b, n := nightAt(t) n.failing["pg"] = true _, err := b.BackUp(context.Background(), "") must(t, err) must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600)) h := b.Health(context.Background()) if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") { t.Fatalf("%s: %q", h.State, h.Why) } } func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) { b, _ := nightAt(t) if h := b.Health(context.Background()); h.State != health.StateHealthy { t.Fatalf("before any night: %s: %s", h.State, h.Why) } } // A module no longer declared here is not held against the machine for a night it failed before. func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) { b, n := nightAt(t) n.failing["mail"] = true _, err := b.BackUp(context.Background(), "") must(t, err) must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600)) if h := b.Health(context.Background()); h.State != health.StateHealthy { t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why) } } // backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning. func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) { b, n := nightAt(t) n.thinning = "Fatal: repository is already locked" _, err := b.BackUp(context.Background(), "") must(t, err) listed, err := b.BackedUp(context.Background(), "pg") must(t, err) if len(listed) != 1 { t.Fatalf("listed %+v", listed) } raw, _ := json.Marshal(listed[0]) var got struct { Retention map[string]any `json:"retention"` Thinned map[string]any `json:"thinned"` } must(t, json.Unmarshal(raw, &got)) if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil { t.Errorf("retention %v, want 14/8/6 not refused", got.Retention) } if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") { t.Errorf("thinned %v, want the failure", got.Thinned) } must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600)) listed, err = b.BackedUp(context.Background(), "pg") must(t, err) raw, _ = json.Marshal(listed[0]) got.Retention = nil must(t, json.Unmarshal(raw, &got)) if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" { t.Errorf("retention %v, want it refused naming keep-daily", got.Retention) } } // The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at // a time across both, never two nights or a night and a restore over one repository. func TestANightWaitsForAnotherProcesssNight(t *testing.T) { b, _ := nightAt(t) release, err := b.hold(nightLock) must(t, err) done := make(chan struct{}) go func() { other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet} _, _ = other.BackUp(context.Background(), "") close(done) }() select { case <-done: t.Fatal("a night ran while another held the repository") case <-time.After(200 * time.Millisecond): } release() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("the night did not run once the other let go") } } // The manifest runs the night and the measuring as processes of their own, as the operator account like the // tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as // before), each given where everything is; the night judged by backup_health, the measuring by its unit — // a tool check only beside a check of another kind (ADR 0240 rule 2). func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { raw, err := os.ReadFile("../../module.json") must(t, err) var m struct { Tools []string `json:"tools"` Resources []map[string]any `json:"resources"` Build struct { Artifacts []struct { Name string `json:"name"` Env map[string]string `json:"env"` } `json:"artifacts"` } `json:"build"` } must(t, json.Unmarshal(raw, &m)) served := map[string]bool{} for _, tool := range tools(&Backups{}) { served[tool.Name] = true } for _, tool := range m.Tools { if !served[tool] { t.Errorf("the manifest lists %s, which the bundle does not serve", tool) } } env := m.Build.Artifacts[0].Env want := map[string]struct{ run, kind, tool string }{ "night": {"night", "tool", "backup_health"}, "measure": {"measure", "unit", ""}, } for _, r := range m.Resources { w, ok := want[r["id"].(string)] if !ok || r["type"] != "process" { continue } delete(want, r["id"].(string)) run, _ := json.Marshal(r["run"]) if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name { t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"]) } if r["user"] != "${machine:account}" { t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"]) } penv, _ := r["env"].(map[string]any) for k, v := range env { if penv[k] != v { t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v) } } h, _ := r["health"].(map[string]any) if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) { t.Errorf("%s is judged by %v", r["id"], h) } if w.tool != "" && !slicesContains(m.Tools, w.tool) { t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool) } } for id := range want { t.Errorf("no process %s", id) } } func slicesContains(s []string, v string) bool { for _, x := range s { if x == v { return true } } return false } // A night that never happens leaves no failed night behind: it is said once the machine was up for more than a // day since the newest good night, counted only while it was awake (novox/hq issue 454). func TestANightThatNeverHappensIsUnhealthy(t *testing.T) { good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: now} seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)} h := Judge(seen) if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") { t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why) } // A laptop asleep most of those two days: up 20 hours of them, nothing is overdue. seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)} if h := Judge(seen); h.State != health.StateHealthy { t.Fatalf("up 20 hours: %s: %q", h.State, h.Why) } // The process stopped counting: said, not taken for a night that is on time. seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)} if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { t.Fatalf("a count an hour old: %s: %q", h.State, h.Why) } } func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) { first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC) now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC) a := Awake{}.Count(first, time.Minute, now) a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) { t.Fatalf("counted %+v", a) } a = a.Count(first.Add(24*time.Hour), time.Minute, now) if a.Up != time.Minute { t.Fatalf("a newer good night did not start the count again: %+v", a) } } // What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one // called a success. func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) { good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good} h := Judge(seen) if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") { t.Fatalf("with no thinning recorded: %q", h.Why) } seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""} if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") { t.Fatalf("after a refused thinning: %q", h.Why) } seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24, 10}} if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") { t.Fatalf("after a thinning: %q", h.Why) } } // The lock that cannot be taken is a night that did not run, and recorded as one. func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) { b, _ := nightAt(t) must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write if _, err := b.BackUp(context.Background(), ""); err == nil { t.Fatal("a night ran without its lock") } if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") { t.Fatalf("%s: %q", h.State, h.Why) } } // A file without hand= is one the mesh did not write as this manifest does (novox/hq issue 453): keep-hand-taken-days // has a default, so there is nothing for the operator to set; it says to push the machine. func TestARetentionWithoutHandSaysToPushTheMachine(t *testing.T) { b, _ := nightAt(t) must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600)) h := b.Health(context.Background()) if h.State != health.StateUnhealthy || len(h.Waits) != 0 { t.Fatalf("a file without hand= is %s waiting for %+v: %s", h.State, h.Waits, h.Why) } for _, want := range []string{"hand= line is missing", "mesh-controller.push"} { if !strings.Contains(h.Why, want) { t.Errorf("said %q, want it to hold %q", h.Why, want) } } // An invalid value is still the operator's, naming the setting. must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\nhand=0\n"), 0o600)) if h := b.Health(context.Background()); h.State != health.StateWaits || len(h.Waits) != 1 || h.Waits[0].Setting != "keep-hand-taken-days" { t.Fatalf("hand=0 is %s waiting for %+v", h.State, h.Waits) } } // A machine whose modules declare nothing to back up has no night to miss: up for days, it is not said to // have none happening (novox/hq issue 489). func TestNothingDeclaredHasNoOverdueNight(t *testing.T) { now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) seen := Seen{Retention: InForce{Retention: &TodaysRetention}, Now: now, Awake: &Awake{Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}} if h := Judge(seen); h.State != health.StateHealthy || strings.Contains(h.Why, "No night is happening") { t.Fatalf("nothing declared, up 27 hours: %s: %q", h.State, h.Why) } } // A laptop that slept: the count was last written before it suspended, hours ago by the wall clock, but only // seconds ago by the time the machine was awake. It is not stale until the machine was awake that long; after a // restart of the machine the monotonic clock begins again, and the wall clock judges (novox/hq issue 489). func TestACountWrittenBeforeASleepIsNotStale(t *testing.T) { good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) now := time.Date(2026, 10, 11, 18, 0, 0, 0, time.UTC) seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}}, Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: now, Boot: "b1", Mono: 5*time.Hour + 30*time.Second} seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b1", Mono: 5 * time.Hour} if h := Judge(seen); h.State != health.StateHealthy { t.Fatalf("counted 30 seconds awake ago, 8 hours asleep: %s: %q", h.State, h.Why) } seen.Awake.Mono = 4 * time.Hour if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { t.Fatalf("counted an hour awake ago: %s: %q", h.State, h.Why) } seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b0", Mono: 5 * time.Hour} if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") { t.Fatalf("counted before the machine restarted, 8 hours ago: %s: %q", h.State, h.Why) } } // The machine's monotonic clock and its boot read on this machine. func TestTheMachinesClockReads(t *testing.T) { boot, mono := machineClock() if boot == "" || mono <= 0 { t.Fatalf("boot %q, monotonic %s", boot, mono) } }