jschoubben
7211227c76
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
404 lines
16 KiB
Go
404 lines
16 KiB
Go
package main
|
|
|
|
// The module says how it is healthy (novox/hq ADR 0240, issue 454): a failed backup, a failed thinning and
|
|
// a refused retention each raise a condition naming the module and what to do, instead of a journal line
|
|
// nobody reads until a disk fills.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
// nightly is a fake restic and fake stores: a module named in failing fails its dump, and forget fails
|
|
// with thinning when it is not empty.
|
|
type nightly struct {
|
|
failing map[string]bool
|
|
thinning string
|
|
}
|
|
|
|
func (n *nightly) run(_ context.Context, name string, args ...string) (string, error) {
|
|
switch {
|
|
case name == "sh" && n.failing[strings.TrimPrefix(args[1], "dump-")]:
|
|
return "", errors.New("the dump failed: no space left on device")
|
|
case name == "restic" && args[5] == "backup":
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
case name == "restic" && args[5] == "forget" && n.thinning != "":
|
|
return "", errors.New(n.thinning)
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func nightAt(t *testing.T) (*Backups, *nightly) {
|
|
t.Helper()
|
|
n := &nightly{failing: map[string]bool{}}
|
|
b := &Backups{Where: placed(t, "# pg\nrun dump-pg\npath /\n# mail\nrun dump-mail\npath /\n"), Run: n.run,
|
|
Now: func() time.Time { return time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC) }, Say: quiet}
|
|
return b, n
|
|
}
|
|
|
|
func TestAFailedBackupIsUnhealthyNamingTheModuleAndWhatToDo(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.failing["mail"] = true
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy {
|
|
t.Fatalf("a failed backup is %s: %s", h.State, h.Why)
|
|
}
|
|
for _, want := range []string{"mail was NOT backed up", "no space left on device", `node-backup.now {"module": "mail"}`} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
if strings.Contains(h.Why, "pg was NOT") {
|
|
t.Errorf("pg was backed up and is said as failed: %q", h.Why)
|
|
}
|
|
|
|
// Backed up again, it clears.
|
|
n.failing["mail"] = false
|
|
_, err = b.BackUp(context.Background(), "mail")
|
|
must(t, err)
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("backed up again, it is %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestANightThatDidNotRunIsUnhealthy(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
b.Run = func(_ context.Context, name string, args ...string) (string, error) {
|
|
return "", errors.New("Fatal: wrong password or no key found")
|
|
}
|
|
if _, err := b.BackUp(context.Background(), ""); err == nil {
|
|
t.Fatal("a repository that will not open ran a night")
|
|
}
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") || !strings.Contains(h.Why, "wrong password") {
|
|
t.Fatalf("a night that did not run: %s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestAFailedThinningIsUnhealthyAndSaysEveryRestorePointIsKept(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.thinning = "Fatal: unable to create lock in backend: repository is already locked exclusively"
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy {
|
|
t.Fatalf("a failed thinning is %s: %s", h.State, h.Why)
|
|
}
|
|
for _, want := range []string{"Thinning the restore points failed", "already locked", "every restore point is kept", "node-backup.now"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
n.thinning = ""
|
|
_, err = b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("thinned again, it is %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// A setting the night refuses is the operator's to give again: the module waits for it, naming it (ADR 0283),
|
|
// and the controller says it as the module's needs-operator condition with the act.
|
|
func TestARefusedRetentionSettingWaitsForTheSettingItNames(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=0\nmonthly=6\n"), 0o600))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateWaits {
|
|
t.Fatalf("a refused setting is %s: %s", h.State, h.Why)
|
|
}
|
|
if len(h.Waits) != 1 || h.Waits[0].Setting != "keep-weekly" || h.Waits[0].Check() != nil {
|
|
t.Fatalf("waits %+v, want one well-formed wait for keep-weekly", h.Waits)
|
|
}
|
|
for _, want := range []string{"keep-weekly", `"0"`, "nothing is thinned", "every restore point is kept"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A file that is missing, or that the mesh wrote in a form this program does not read, is no setting the
|
|
// operator gave: it is a fault, and says to push the machine.
|
|
func TestAMissingRetentionFileIsUnhealthy(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.Remove(filepath.Join(b.Where.State, RetentionFile)))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy {
|
|
t.Fatalf("a missing retention file is %s: %s", h.State, h.Why)
|
|
}
|
|
for _, want := range []string{"retention.conf is missing", "nothing is thinned", "mesh-controller.push"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A refused setting beside a failed backup: unhealthy, still naming the setting it waits for.
|
|
func TestAFailedBackupBesideARefusedSettingIsUnhealthy(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.failing["pg"] = true
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=x\nweekly=8\nmonthly=6\n"), 0o600))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "pg was NOT backed up") || !strings.Contains(h.Why, "keep-daily") {
|
|
t.Fatalf("%s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestBeforeAnyNightTheModuleIsHealthy(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("before any night: %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// A module no longer declared here is not held against the machine for a night it failed before.
|
|
func TestAModuleNoLongerDeclaredIsNotJudged(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.failing["mail"] = true
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
must(t, os.WriteFile(b.Where.Declared, []byte("# pg\nrun dump-pg\npath /\n"), 0o600))
|
|
if h := b.Health(context.Background()); h.State != health.StateHealthy {
|
|
t.Fatalf("mail is gone and still judged: %s: %s", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// backed-up says the retention in force, and whether it was refused, beside each module, and the last thinning.
|
|
func TestBackedUpSaysTheRetentionInForceAndWhetherItWasRefused(t *testing.T) {
|
|
b, n := nightAt(t)
|
|
n.thinning = "Fatal: repository is already locked"
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
listed, err := b.BackedUp(context.Background(), "pg")
|
|
must(t, err)
|
|
if len(listed) != 1 {
|
|
t.Fatalf("listed %+v", listed)
|
|
}
|
|
raw, _ := json.Marshal(listed[0])
|
|
var got struct {
|
|
Retention map[string]any `json:"retention"`
|
|
Thinned map[string]any `json:"thinned"`
|
|
}
|
|
must(t, json.Unmarshal(raw, &got))
|
|
if got.Retention["daily"] != 14.0 || got.Retention["weekly"] != 8.0 || got.Retention["monthly"] != 6.0 || got.Retention["refused"] != nil {
|
|
t.Errorf("retention %v, want 14/8/6 not refused", got.Retention)
|
|
}
|
|
if got.Thinned["ok"] != false || !strings.Contains(got.Thinned["error"].(string), "already locked") {
|
|
t.Errorf("thinned %v, want the failure", got.Thinned)
|
|
}
|
|
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=0\nweekly=8\nmonthly=6\n"), 0o600))
|
|
listed, err = b.BackedUp(context.Background(), "pg")
|
|
must(t, err)
|
|
raw, _ = json.Marshal(listed[0])
|
|
got.Retention = nil
|
|
must(t, json.Unmarshal(raw, &got))
|
|
if r, _ := got.Retention["refused"].(string); !strings.Contains(r, "keep-daily") || got.Retention["setting"] != "keep-daily" {
|
|
t.Errorf("retention %v, want it refused naming keep-daily", got.Retention)
|
|
}
|
|
}
|
|
|
|
// The night runs in a process of its own, beside the tool runtime that serves `now` and `restore`: one thing at
|
|
// a time across both, never two nights or a night and a restore over one repository.
|
|
func TestANightWaitsForAnotherProcesssNight(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
release, err := b.hold(nightLock)
|
|
must(t, err)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
other := &Backups{Where: b.Where, Run: b.Run, Now: b.Now, Say: quiet}
|
|
_, _ = other.BackUp(context.Background(), "")
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
t.Fatal("a night ran while another held the repository")
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
release()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the night did not run once the other let go")
|
|
}
|
|
}
|
|
|
|
// The manifest runs the night and the measuring as processes of their own, as the operator account like the
|
|
// tool runtime (so the files they write stay readable to `backed-up`, and root's acts go through sudo as
|
|
// before), each given where everything is; the night judged by backup_health, the measuring by its unit —
|
|
// a tool check only beside a check of another kind (ADR 0240 rule 2).
|
|
func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
|
|
raw, err := os.ReadFile("../../module.json")
|
|
must(t, err)
|
|
var m struct {
|
|
Tools []string `json:"tools"`
|
|
Resources []map[string]any `json:"resources"`
|
|
Build struct {
|
|
Artifacts []struct {
|
|
Name string `json:"name"`
|
|
Env map[string]string `json:"env"`
|
|
} `json:"artifacts"`
|
|
} `json:"build"`
|
|
}
|
|
must(t, json.Unmarshal(raw, &m))
|
|
served := map[string]bool{}
|
|
for _, tool := range tools(&Backups{}) {
|
|
served[tool.Name] = true
|
|
}
|
|
for _, tool := range m.Tools {
|
|
if !served[tool] {
|
|
t.Errorf("the manifest lists %s, which the bundle does not serve", tool)
|
|
}
|
|
}
|
|
env := m.Build.Artifacts[0].Env
|
|
want := map[string]struct{ run, kind, tool string }{
|
|
"night": {"night", "tool", "backup_health"},
|
|
"measure": {"measure", "unit", ""},
|
|
}
|
|
for _, r := range m.Resources {
|
|
w, ok := want[r["id"].(string)]
|
|
if !ok || r["type"] != "process" {
|
|
continue
|
|
}
|
|
delete(want, r["id"].(string))
|
|
run, _ := json.Marshal(r["run"])
|
|
if string(run) != `["./restic-backups","`+w.run+`"]` || r["artifact"] != m.Build.Artifacts[0].Name {
|
|
t.Errorf("%s runs %s from %v", r["id"], run, r["artifact"])
|
|
}
|
|
if r["user"] != "${machine:account}" {
|
|
t.Errorf("%s runs as %v, not the operator account", r["id"], r["user"])
|
|
}
|
|
penv, _ := r["env"].(map[string]any)
|
|
for k, v := range env {
|
|
if penv[k] != v {
|
|
t.Errorf("%s is given %s=%v, the bundle %s", r["id"], k, penv[k], v)
|
|
}
|
|
}
|
|
h, _ := r["health"].(map[string]any)
|
|
if h["kind"] != w.kind || (w.tool != "" && h["tool"] != w.tool) {
|
|
t.Errorf("%s is judged by %v", r["id"], h)
|
|
}
|
|
if w.tool != "" && !slicesContains(m.Tools, w.tool) {
|
|
t.Errorf("%s's check asks %s, which the manifest does not list among its tools", r["id"], w.tool)
|
|
}
|
|
}
|
|
for id := range want {
|
|
t.Errorf("no process %s", id)
|
|
}
|
|
}
|
|
|
|
func slicesContains(s []string, v string) bool {
|
|
for _, x := range s {
|
|
if x == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// A night that never happens leaves no failed night behind: it is said once the machine was up for more than a
|
|
// day since the newest good night, counted only while it was awake (novox/hq issue 454).
|
|
func TestANightThatNeverHappensIsUnhealthy(t *testing.T) {
|
|
good := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
|
|
now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC)
|
|
seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
|
|
Nights: map[string]Night{"pg": {OK: true, At: good}},
|
|
Retention: InForce{Retention: &TodaysRetention}, Now: now}
|
|
|
|
seen.Awake = &Awake{From: good, Up: 27 * time.Hour, Looked: now.Add(-time.Minute)}
|
|
h := Judge(seen)
|
|
if h.State != health.StateUnhealthy || !strings.Contains(h.Why, "No night is happening") || !strings.Contains(h.Why, "27 hours") {
|
|
t.Fatalf("up 27 hours without a night: %s: %q", h.State, h.Why)
|
|
}
|
|
|
|
// A laptop asleep most of those two days: up 20 hours of them, nothing is overdue.
|
|
seen.Awake = &Awake{From: good, Up: 20 * time.Hour, Looked: now.Add(-time.Minute)}
|
|
if h := Judge(seen); h.State != health.StateHealthy {
|
|
t.Fatalf("up 20 hours: %s: %q", h.State, h.Why)
|
|
}
|
|
|
|
// The process stopped counting: said, not taken for a night that is on time.
|
|
seen.Awake = &Awake{From: good, Up: time.Hour, Looked: now.Add(-time.Hour)}
|
|
if h := Judge(seen); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "last counted") {
|
|
t.Fatalf("a count an hour old: %s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
func TestTheCountRunsOnlyWhileAwakeAndStartsAgainAtAGoodNight(t *testing.T) {
|
|
first := time.Date(2026, 10, 9, 3, 0, 0, 0, time.UTC)
|
|
now := time.Date(2026, 10, 10, 3, 0, 0, 0, time.UTC)
|
|
a := Awake{}.Count(first, time.Minute, now)
|
|
a = a.Count(first, 2*time.Hour, now) // a gap of a night asleep counts only what was awake
|
|
if a.Up != 2*time.Hour+time.Minute || !a.From.Equal(first) {
|
|
t.Fatalf("counted %+v", a)
|
|
}
|
|
a = a.Count(first.Add(24*time.Hour), time.Minute, now)
|
|
if a.Up != time.Minute {
|
|
t.Fatalf("a newer good night did not start the count again: %+v", a)
|
|
}
|
|
}
|
|
|
|
// What a healthy answer says is what was seen: no thinning claimed before one is recorded, nor a refused one
|
|
// called a success.
|
|
func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) {
|
|
good := time.Date(2026, 10, 11, 3, 0, 0, 0, time.UTC)
|
|
seen := Seen{Declared: []Declared{{Module: "pg", Paths: []string{"/"}}},
|
|
Nights: map[string]Night{"pg": {OK: true, At: good}}, Retention: InForce{Retention: &TodaysRetention}, Now: good}
|
|
h := Judge(seen)
|
|
if h.State != health.StateHealthy || strings.Contains(h.Why, "succeeded") || !strings.Contains(h.Why, "No thinning is recorded yet") {
|
|
t.Fatalf("with no thinning recorded: %q", h.Why)
|
|
}
|
|
seen.Run.Thinned = &Thinning{At: good, Refused: "the keep-daily setting is \"0\""}
|
|
if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") {
|
|
t.Fatalf("after a refused thinning: %q", h.Why)
|
|
}
|
|
seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24, 10}}
|
|
if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") {
|
|
t.Fatalf("after a thinning: %q", h.Why)
|
|
}
|
|
}
|
|
|
|
// The lock that cannot be taken is a night that did not run, and recorded as one.
|
|
func TestANightWhoseLockCannotBeTakenIsRecorded(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write
|
|
if _, err := b.BackUp(context.Background(), ""); err == nil {
|
|
t.Fatal("a night ran without its lock")
|
|
}
|
|
if h := b.Health(context.Background()); h.State != health.StateUnhealthy || !strings.Contains(h.Why, "did not run") {
|
|
t.Fatalf("%s: %q", h.State, h.Why)
|
|
}
|
|
}
|
|
|
|
// A file without hand= is one the mesh did not write as this manifest does (novox/hq issue 453): keep-hand-taken-days
|
|
// has a default, so there is nothing for the operator to set; it says to push the machine.
|
|
func TestARetentionWithoutHandSaysToPushTheMachine(t *testing.T) {
|
|
b, _ := nightAt(t)
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600))
|
|
h := b.Health(context.Background())
|
|
if h.State != health.StateUnhealthy || len(h.Waits) != 0 {
|
|
t.Fatalf("a file without hand= is %s waiting for %+v: %s", h.State, h.Waits, h.Why)
|
|
}
|
|
for _, want := range []string{"hand= line is missing", "mesh-controller.push"} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("said %q, want it to hold %q", h.Why, want)
|
|
}
|
|
}
|
|
// An invalid value is still the operator's, naming the setting.
|
|
must(t, os.WriteFile(filepath.Join(b.Where.State, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\nhand=0\n"), 0o600))
|
|
if h := b.Health(context.Background()); h.State != health.StateWaits || len(h.Waits) != 1 || h.Waits[0].Setting != "keep-hand-taken-days" {
|
|
t.Fatalf("hand=0 is %s waiting for %+v", h.State, h.Waits)
|
|
}
|
|
}
|