jschoubben
cbe9fa7a2c
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
A hung night, a lock held for hours or a lock that cannot be taken left no failed night behind, so it stayed healthy. The night's process now counts how long the machine was up since its newest good night, by the monotonic clock so a sleeping laptop is not blamed, and past 26 hours it is raised. The retention file is written 0644 and root's instead of read through sudo: the account reads it and still cannot change it. An unreadable file names its cause, and the healthy answer says only what was seen.
383 lines
15 KiB
Go
383 lines
15 KiB
Go
package main
|
|
|
|
// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and
|
|
// a refused retention each raise a condition naming the module and what to do, instead of a journal line
|
|
// nobody reads until a disk fills.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails
|
|
// with thinning when it is not empty.
|
|
type nightly struct {
|
|
failing map[string]bool
|
|
thinning string
|
|
}
|
|
|
|
func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) {
|
|
switch {
|
|
case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]:
|
|
return "", errors.New("the dump failed: no space left on device")
|
|
case name == "restic" && args[5] == "backup":
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
case name == "restic" && args[5] == "forget" && n.thinning != "":
|
|
return "", errors.New(n.thinning)
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func nightAt(t *testing.T) (*Backups, *nightly) {
|
|
t.Helper()
|
|
n := &nightly{failing: map[string]bool{}}
|
|
b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run,
|
|
Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet}
|
|
return b, n
|
|
}
|
|
|
|
func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.failing["mail"] = true
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy {
|
|
t.Fatalf("a failed backup is %s: %s", h.State, h.Why)
|
|
}
|
|
for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
if strings.Contains(h.Why, "pg was NOT") {
|
|
t.Errorf("pg was backed up and is said as failed: %q", h.Why)
|
|
}
|
|
|
|
// Backed up again, it clears.
|
|
n.failing["mail"] = false
|
|
_, err = b.BackUp(context.Background(), "mail")
|
|
must(t, err)
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("backed up again, it is %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestANightThatDidNotRunIsUnhealthy(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
b.Run = func(_ context.Context, name string, args ...string) (string, error) {
|
|
return "", errors.New("Fatal: wrong password or no key found")
|
|
}
|
|
if _, err := b.BackUp(context.Background(), ""); err == nil {
|
|
t.Fatal("a repository that will not open ran a night")
|
|
}
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") {
|
|
t.Fatalf("a night that did not run: %s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively"
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy {
|
|
t.Fatalf("a failed thinning is %s: %s", h.State, h.Why)
|
|
}
|
|
for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
n.thinning = ""
|
|
_, err = b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("thinned again, it is %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283),
|
|
// and the controller says it as the module's needs-operator condition with the act.
|
|
func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateWaits {
|
|
t.Fatalf("a refused setting is %s: %s", h.State, h.Why)
|
|
}
|
|
if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil {
|
|
t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits)
|
|
}
|
|
for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the
|
|
// operator gave: it is a fault, and says to push the machine.
|
|
func TestAMissingRetentionFileIsUnhealthy(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile)))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy {
|
|
t.Fatalf("a missing retention file is %s: %s", h.State, h.Why)
|
|
}
|
|
for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for.
|
|
func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.failing["pg"] = true
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") {
|
|
t.Fatalf("%s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("before any night: %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// A module no longer declared here is not held against the machine for a night it failed before.
|
|
func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.failing["mail"] = true
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600))
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning.
|
|
func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.thinning = "Fatal: repository is already locked"
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
listed, err := b.BackedUp(context.Background(), "pg")
|
|
must(t, err)
|
|
if len(listed) != 1 {
|
|
t.Fatalf("listed %+v", listed)
|
|
}
|
|
raw, _ := json.Marshal(listed[0])
|
|
var got struct {
|
|
Retention map[string]any `json:"retention"`
|
|
Thinned map[string]any `json:"thinned"`
|
|
}
|
|
must(t, json.Unmarshal(raw, &got))
|
|
if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil {
|
|
t.Errorf("retention %v, want 14/8/6 not refused", got.Retention)
|
|
}
|
|
if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") {
|
|
t.Errorf("thinned %v, want the failure", got.Thinned)
|
|
}
|
|
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600))
|
|
listed, err = b.BackedUp(context.Background(), "pg")
|
|
must(t, err)
|
|
raw, _ = json.Marshal(listed[0])
|
|
got.Retention = nil
|
|
must(t, json.Unmarshal(raw, &got))
|
|
if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" {
|
|
t.Errorf("retention %v, want it refused naming keep-daily", got.Retention)
|
|
}
|
|
}
|
|
|
|
// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at
|
|
// a time across both, never two nights or a night and a restore over one repository.
|
|
func TestANightWaitsForAnotherProcesssNight(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
release, err := b.hold(nightLock)
|
|
must(t, err)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet}
|
|
_, _ = other.BackUp(context.Background(), "")
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
t.Fatal("a night ran while another held the repository")
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
release()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the night did not run once the other let go")
|
|
}
|
|
}
|
|
|
|
// The manifest runs the night and the measuring as processes of their own, as the operator account like the
|
|
// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as
|
|
// before), each given where everything is; the night judged by backup_health, the measuring by its unit —
|
|
// a tool check only beside a check of another kind (ADR 0240 rule 2).
|
|
func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
|
|
raw, err := os.ReadFile("../../module.json")
|
|
must(t, err)
|
|
var m struct {
|
|
Tools []string `json:"tools"`
|
|
Resources []map[string]any `json:"resources"`
|
|
Build struct {
|
|
Artifacts []struct {
|
|
Name string `json:"name"`
|
|
Env map[string]string `json:"env"`
|
|
} `json:"artifacts"`
|
|
} `json:"build"`
|
|
}
|
|
must(t, json.Unmarshal(raw, &m))
|
|
served := map[string]bool{}
|
|
for _, tool := range tools(&Backups{}) {
|
|
served[tool.Name] = true
|
|
}
|
|
for _, tool := range m.Tools {
|
|
if !served[tool] {
|
|
t.Errorf("the manifest lists %s, which the bundle does not serve", tool)
|
|
}
|
|
}
|
|
env := m.Build.Artifacts[0].Env
|
|
want := map[string]struct{ run, kind, tool string }{
|
|
"night": {"night", "tool", "backup_health"},
|
|
"measure": {"measure", "unit", ""},
|
|
}
|
|
for _, r := range m.Resources {
|
|
w, ok := want[r["id"].(string)]
|
|
if !ok || r["type"] != "process" {
|
|
continue
|
|
}
|
|
delete(want, r["id"].(string))
|
|
run, _ := json.Marshal(r["run"])
|
|
if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name {
|
|
t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"])
|
|
}
|
|
if r["user"] != "${machine:account}" {
|
|
t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"])
|
|
}
|
|
penv, _ := r["env"].(map[string]any)
|
|
for k, v := range env {
|
|
if penv[k] != v {
|
|
t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v)
|
|
}
|
|
}
|
|
h, _ := r["health"].(map[string]any)
|
|
if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) {
|
|
t.Errorf("%s is judged by %v", r["id"], h)
|
|
}
|
|
if w.tool != "" && !slicesContains(m.Tools, w.tool) {
|
|
t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool)
|
|
}
|
|
}
|
|
for id := range want {
|
|
t.Errorf("no process %s", id)
|
|
}
|
|
}
|
|
|
|
func slicesContains(s []string, v string) bool {
|
|
for _, x := range s {
|
|
if x == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a
|
|
// day since the newest good night, counted only while it was awake (novox/hq issue 454).
|
|
func TestANightThatNeverHappensIsUnhealthy(t *testing.T) {
|
|
good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
|
|
now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC)
|
|
seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
|
|
Nights: map[string]Night{"pg": {OK: true, At: good}},
|
|
Retention: InForce{Retention: &TodaysRetention}, Now: now}
|
|
|
|
seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}
|
|
h := Judge(seen)
|
|
if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") {
|
|
t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why)
|
|
}
|
|
|
|
// A laptop asleep most of those two days: up 20 hours of them, nothing is overdue.
|
|
seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)}
|
|
if h := Judge(seen); h.State != health.StateHealthy {
|
|
t.Fatalf("up 20 hours: %s: %q", h.State, h.Why)
|
|
}
|
|
|
|
// The process stopped counting: said, not taken for a night that is on time.
|
|
seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)}
|
|
if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
|
|
t.Fatalf("a count an hour old: %s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) {
|
|
first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
|
|
now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC)
|
|
a := Awake{}.Count(first, time.Minute, now)
|
|
a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake
|
|
if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) {
|
|
t.Fatalf("counted %+v", a)
|
|
}
|
|
a = a.Count(first.Add(24*time.Hour), time.Minute, now)
|
|
if a.Up != time.Minute {
|
|
t.Fatalf("a newer good night did not start the count again: %+v", a)
|
|
}
|
|
}
|
|
|
|
// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one
|
|
// called a success.
|
|
func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) {
|
|
good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC)
|
|
seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
|
|
Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good}
|
|
h := Judge(seen)
|
|
if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") {
|
|
t.Fatalf("with no thinning recorded: %q", h.Why)
|
|
}
|
|
seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""}
|
|
if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") {
|
|
t.Fatalf("after a refused thinning: %q", h.Why)
|
|
}
|
|
seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24}}
|
|
if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") {
|
|
t.Fatalf("after a thinning: %q", h.Why)
|
|
}
|
|
}
|
|
|
|
// The lock that cannot be taken is a night that did not run, and recorded as one.
|
|
func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write
|
|
if _, err := b.BackUp(context.Background(), ""); err == nil {
|
|
t.Fatal("a night ran without its lock")
|
|
}
|
|
if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") {
|
|
t.Fatalf("%s: %q", h.State, h.Why)
|
|
}
|
|
}
|