mesh-catalog/modules/restic/cmd/restic-backups/health_test.go

package main

// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and
// a refused retention each raise a condition naming the module and what to do, instead of a journal line
// nobody reads until a disk fills.

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/health"
)

// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails
// with thinning when it is not empty.
type nightly struct {
	failing  map[string]bool
	thinning string
}

func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) {
	switch {
	case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]:
		return "", errors.New("the dump failed: no space left on device")
	case name == "restic" && args[5] == "backup":
		return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
	case name == "restic" && args[5] == "forget" && n.thinning != "":
		return "", errors.New(n.thinning)
	}
	return "", nil
}

func nightAt(t *testing.T) (*Backups, *nightly) {
	t.Helper()
	n := &nightly{failing: map[string]bool{}}
	b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run,
		Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet}
	return b, n
}

func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) {
	b, n := nightAt(t)
	n.failing["mail"] = true
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	h := b.Health(context.Background())
	if h.State != health.StateUnhealthy {
		t.Fatalf("a failed backup is %s: %s", h.State, h.Why)
	}
	for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} {
		if !strings.Contains(h.Why, want) {
			t.Errorf("said %q, want it to hold %q", h.Why, want)
		}
	}
	if strings.Contains(h.Why, "pg was NOT") {
		t.Errorf("pg was backed up and is said as failed: %q", h.Why)
	}

	// Backed up again, it clears.
	n.failing["mail"] = false
	_, err = b.BackUp(context.Background(), "mail")
	must(t, err)
	if h := b.Health(context.Background()); h.State != health.StateHealthy {
		t.Fatalf("backed up again, it is %s: %s", h.State, h.Why)
	}
}

func TestANightThatDidNotRunIsUnhealthy(t *testing.T) {
	b, _ := nightAt(t)
	b.Run = func(_ context.Context, name string, args ...string) (string, error) {
		return "", errors.New("Fatal: wrong password or no key found")
	}
	if _, err := b.BackUp(context.Background(), ""); err == nil {
		t.Fatal("a repository that will not open ran a night")
	}
	h := b.Health(context.Background())
	if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") {
		t.Fatalf("a night that did not run: %s: %q", h.State, h.Why)
	}
}

func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) {
	b, n := nightAt(t)
	n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively"
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	h := b.Health(context.Background())
	if h.State != health.StateUnhealthy {
		t.Fatalf("a failed thinning is %s: %s", h.State, h.Why)
	}
	for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} {
		if !strings.Contains(h.Why, want) {
			t.Errorf("said %q, want it to hold %q", h.Why, want)
		}
	}
	n.thinning = ""
	_, err = b.BackUp(context.Background(), "")
	must(t, err)
	if h := b.Health(context.Background()); h.State != health.StateHealthy {
		t.Fatalf("thinned again, it is %s: %s", h.State, h.Why)
	}
}

// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283),
// and the controller says it as the module's needs-operator condition with the act.
func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) {
	b, _ := nightAt(t)
	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600))
	h := b.Health(context.Background())
	if h.State != health.StateWaits {
		t.Fatalf("a refused setting is %s: %s", h.State, h.Why)
	}
	if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil {
		t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits)
	}
	for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} {
		if !strings.Contains(h.Why, want) {
			t.Errorf("said %q, want it to hold %q", h.Why, want)
		}
	}
}

// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the
// operator gave: it is a fault, and says to push the machine.
func TestAMissingRetentionFileIsUnhealthy(t *testing.T) {
	b, _ := nightAt(t)
	must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile)))
	h := b.Health(context.Background())
	if h.State != health.StateUnhealthy {
		t.Fatalf("a missing retention file is %s: %s", h.State, h.Why)
	}
	for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} {
		if !strings.Contains(h.Why, want) {
			t.Errorf("said %q, want it to hold %q", h.Why, want)
		}
	}
}

// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for.
func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) {
	b, n := nightAt(t)
	n.failing["pg"] = true
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600))
	h := b.Health(context.Background())
	if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") {
		t.Fatalf("%s: %q", h.State, h.Why)
	}
}

func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) {
	b, _ := nightAt(t)
	if h := b.Health(context.Background()); h.State != health.StateHealthy {
		t.Fatalf("before any night: %s: %s", h.State, h.Why)
	}
}

// A module no longer declared here is not held against the machine for a night it failed before.
func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) {
	b, n := nightAt(t)
	n.failing["mail"] = true
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600))
	if h := b.Health(context.Background()); h.State != health.StateHealthy {
		t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why)
	}
}

// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning.
func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) {
	b, n := nightAt(t)
	n.thinning = "Fatal: repository is already locked"
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	listed, err := b.BackedUp(context.Background(), "pg")
	must(t, err)
	if len(listed) != 1 {
		t.Fatalf("listed %+v", listed)
	}
	raw, _ := json.Marshal(listed[0])
	var got struct {
		Retention map[string]any `json:"retention"`
		Thinned   map[string]any `json:"thinned"`
	}
	must(t, json.Unmarshal(raw, &got))
	if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil {
		t.Errorf("retention %v, want 14/8/6 not refused", got.Retention)
	}
	if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") {
		t.Errorf("thinned %v, want the failure", got.Thinned)
	}

	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600))
	listed, err = b.BackedUp(context.Background(), "pg")
	must(t, err)
	raw, _ = json.Marshal(listed[0])
	got.Retention = nil
	must(t, json.Unmarshal(raw, &got))
	if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" {
		t.Errorf("retention %v, want it refused naming keep-daily", got.Retention)
	}
}

// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at
// a time across both, never two nights or a night and a restore over one repository.
func TestANightWaitsForAnotherProcesssNight(t *testing.T) {
	b, _ := nightAt(t)
	release, err := b.hold(nightLock)
	must(t, err)
	done := make(chan struct{})
	go func() {
		other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet}
		_, _ = other.BackUp(context.Background(), "")
		close(done)
	}()
	select {
	case <-done:
		t.Fatal("a night ran while another held the repository")
	case <-time.After(200 * time.Millisecond):
	}
	release()
	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("the night did not run once the other let go")
	}
}

// The manifest runs the night and the measuring as processes of their own, as the operator account like the
// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as
// before), each given where everything is; the night judged by backup_health, the measuring by its unit —
// a tool check only beside a check of another kind (ADR 0240 rule 2).
func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
	raw, err := os.ReadFile("../../module.json")
	must(t, err)
	var m struct {
		Tools     []string         `json:"tools"`
		Resources []map[string]any `json:"resources"`
		Build     struct {
			Artifacts []struct {
				Name string            `json:"name"`
				Env  map[string]string `json:"env"`
			} `json:"artifacts"`
		} `json:"build"`
	}
	must(t, json.Unmarshal(raw, &m))
	served := map[string]bool{}
	for _, tool := range tools(&Backups{}) {
		served[tool.Name] = true
	}
	for _, tool := range m.Tools {
		if !served[tool] {
			t.Errorf("the manifest lists %s, which the bundle does not serve", tool)
		}
	}
	env := m.Build.Artifacts[0].Env
	want := map[string]struct{ run, kind, tool string }{
		"night": {"night", "unit", ""},
		"awake": {"awake", "tool", "backup_health"},
	}
	for _, r := range m.Resources {
		w, ok := want[r["id"].(string)]
		if !ok || r["type"] != "process" {
			continue
		}
		delete(want, r["id"].(string))
		run, _ := json.Marshal(r["run"])
		if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name {
			t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"])
		}
		if r["user"] != "${machine:account}" {
			t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"])
		}
		penv, _ := r["env"].(map[string]any)
		for k, v := range env {
			if penv[k] != v {
				t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v)
			}
		}
		h, _ := r["health"].(map[string]any)
		if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) {
			t.Errorf("%s is judged by %v", r["id"], h)
		}
		if w.tool != "" && !slicesContains(m.Tools, w.tool) {
			t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool)
		}
	}
	for id := range want {
		t.Errorf("no process %s", id)
	}
}

func slicesContains(s []string, v string) bool {
	for _, x := range s {
		if x == v {
			return true
		}
	}
	return false
}

// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a
// day since the newest good night, counted only while it was awake (novox/hq issue 454).
func TestANightThatNeverHappensIsUnhealthy(t *testing.T) {
	good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
	now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC)
	seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
		Nights:    map[string]Night{"pg": {OK: true, At: good}},
		Retention: InForce{Retention: &TodaysRetention}, Now: now}

	seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}
	h := Judge(seen)
	if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") {
		t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why)
	}

	// A laptop asleep most of those two days: up 20 hours of them, nothing is overdue.
	seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)}
	if h := Judge(seen); h.State != health.StateHealthy {
		t.Fatalf("up 20 hours: %s: %q", h.State, h.Why)
	}

	// The process stopped counting: said, not taken for a night that is on time.
	seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)}
	if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
		t.Fatalf("a count an hour old: %s: %q", h.State, h.Why)
	}
}

func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) {
	first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
	now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC)
	a := Awake{}.Count(first, time.Minute, now)
	a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake
	if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) {
		t.Fatalf("counted %+v", a)
	}
	a = a.Count(first.Add(24*time.Hour), time.Minute, now)
	if a.Up != time.Minute {
		t.Fatalf("a newer good night did not start the count again: %+v", a)
	}
}

// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one
// called a success.
func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) {
	good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC)
	seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
		Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good}
	h := Judge(seen)
	if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") {
		t.Fatalf("with no thinning recorded: %q", h.Why)
	}
	seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""}
	if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") {
		t.Fatalf("after a refused thinning: %q", h.Why)
	}
	seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24, 10}}
	if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") {
		t.Fatalf("after a thinning: %q", h.Why)
	}
}

// The lock that cannot be taken is a night that did not run, and recorded as one.
func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) {
	b, _ := nightAt(t)
	must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write
	if _, err := b.BackUp(context.Background(), ""); err == nil {
		t.Fatal("a night ran without its lock")
	}
	if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") {
		t.Fatalf("%s: %q", h.State, h.Why)
	}
}

// A file without hand= is one the mesh did not write as this manifest does (novox/hq issue 453): keep-hand-taken-days
// has a default, so there is nothing for the operator to set; it says to push the machine.
func TestARetentionWithoutHandSaysToPushTheMachine(t *testing.T) {
	b, _ := nightAt(t)
	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600))
	h := b.Health(context.Background())
	if h.State != health.StateUnhealthy || len(h.Waits) != 0 {
		t.Fatalf("a file without hand= is %s waiting for %+v: %s", h.State, h.Waits, h.Why)
	}
	for _, want := range []string{"hand= line is missing", "mesh-controller.push"} {
		if !strings.Contains(h.Why, want) {
			t.Errorf("said %q, want it to hold %q", h.Why, want)
		}
	}
	// An invalid value is still the operator's, naming the setting.
	must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\nhand=0\n"), 0o600))
	if h := b.Health(context.Background()); h.State != health.StateWaits || len(h.Waits) != 1 || h.Waits[0].Setting != "keep-hand-taken-days" {
		t.Fatalf("hand=0 is %s waiting for %+v", h.State, h.Waits)
	}
}

// A machine whose modules declare nothing to back up has no night to miss: up for days, it is not said to
// have none happening (novox/hq issue 489).
func TestNothingDeclaredHasNoOverdueNight(t *testing.T) {
	now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC)
	seen := Seen{Retention: InForce{Retention: &TodaysRetention}, Now: now,
		Awake: &Awake{Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}}
	if h := Judge(seen); h.State != health.StateHealthy || strings.Contains(h.Why, "No night is happening") {
		t.Fatalf("nothing declared, up 27 hours: %s: %q", h.State, h.Why)
	}
}

// A laptop that slept: the count was last written before it suspended, hours ago by the wall clock, but only
// seconds ago by the time the machine was awake. It is not stale until the machine was awake that long; after a
// restart of the machine the monotonic clock begins again, and the wall clock judges (novox/hq issue 489).
func TestACountWrittenBeforeASleepIsNotStale(t *testing.T) {
	good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC)
	now := time.Date(2026, 10, 11, 18, 0, 0, 0, time.UTC)
	seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
		Nights:    map[string]Night{"pg": {OK: true, At: good}},
		Retention: InForce{Retention: &TodaysRetention}, Now: now,
		Boot: "b1", Mono: 5*time.Hour + 30*time.Second}

	seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b1", Mono: 5 * time.Hour}
	if h := Judge(seen); h.State != health.StateHealthy {
		t.Fatalf("counted 30 seconds awake ago, 8 hours asleep: %s: %q", h.State, h.Why)
	}

	seen.Awake.Mono = 4 * time.Hour
	if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
		t.Fatalf("counted an hour awake ago: %s: %q", h.State, h.Why)
	}

	seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-8 * time.Hour), Boot: "b0", Mono: 5 * time.Hour}
	if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
		t.Fatalf("counted before the machine restarted, 8 hours ago: %s: %q", h.State, h.Why)
	}
}

// The machine's monotonic clock and its boot read on this machine.
func TestTheMachinesClockReads(t *testing.T) {
	boot, mono := machineClock()
	if boot == "" || mono <= 0 {
		t.Fatalf("boot %q, monotonic %s", boot, mono)
	}
}