mesh-controller/cmd/mesh-controller/plan_backups_test.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
)

// The backups before a delivery plan asks anything (novox/hq ADR 0404 decisions 2 and 3, issue 1065).

// aModuleWith is a catalogue entry for a module keeping the given data items, on the given machines.
func aModuleWith(t *testing.T, name string, on []string, items ...string) inventory.Entry {
	t.Helper()
	raw := `{"module":"` + name + `","version":"1"`
	if len(items) > 0 {
		raw += `,"data":{"own":[` + strings.Join(items, ",") + `]}`
	}
	raw += `}`
	var m catalogue.Manifest
	if err := json.Unmarshal([]byte(raw), &m); err != nil {
		t.Fatal(err)
	}
	return inventory.Entry{Manifest: m, On: on}
}

func anItem(id, extra string) string {
	if extra != "" {
		extra = "," + extra
	}
	return `{"id":"` + id + `","path":"/x/` + id + `","class":"valuable","why":"w"` + extra + `}`
}

func holders(nodes ...string) []catalogue.Held {
	var out []catalogue.Held
	for _, n := range nodes {
		out = append(out, catalogue.Held{Claim: catalogue.BackupSeat, Node: n, Module: "restic"})
	}
	return out
}

// aFakeBackupSeat stands in for the machines' node-backup holders.
type aFakeBackupSeat struct {
	asked  []string // "machine/module", in order
	nights map[string]*night
	refuse map[string]string
}

func (f *aFakeBackupSeat) install(t *testing.T) {
	t.Helper()
	was, wasRead := askBackupNow, readLastNights
	t.Cleanup(func() { askBackupNow, readLastNights = was, wasRead })
	askBackupNow = func(_ context.Context, machine, module string) error {
		f.asked = append(f.asked, machine+"/"+module)
		if why := f.refuse[machine+"/"+module]; why != "" {
			return fmt.Errorf("%s", why)
		}
		return nil
	}
	readLastNights = func(_ context.Context, machine string) (map[string]*night, error) {
		out := map[string]*night{}
		for k, n := range f.nights {
			if m, mod, _ := strings.Cut(k, "/"); m == machine {
				out[mod] = n
			}
		}
		return out, nil
	}
}

func aPlanOf(modules ...string) *inventory.Plan {
	p := &inventory.Plan{ID: "plan-b", State: inventory.PlanBuilding, Tiers: [][]string{modules},
		Modules: map[string]*inventory.PlanModule{}}
	for _, m := range modules {
		p.Modules[m] = &inventory.PlanModule{}
	}
	return p
}

var backupT0 = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)

// Only a module with an item protected by a backup is asked, and on the machines that run it.
func TestOnlyModulesWithAnItemProtectedByABackupAreAsked(t *testing.T) {
	seat := &aFakeBackupSeat{}
	seat.install(t)
	entries := []inventory.Entry{
		aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", "")),
		aModuleWith(t, "dumped", []string{"ace"}, anItem("b", `"backup":{"dump":"pg_dump","into":"d"}`), anItem("d", "")),
		aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)),
		aModuleWith(t, "unbacked", []string{"ace"}, anItem("e", `"backup":"none"`)),
		aModuleWith(t, "cached", []string{"ace"}, `{"id":"f","path":"/x/f","class":"cache"}`),
		aModuleWith(t, "dataless", []string{"ace"}),
	}
	p := aPlanOf("copied", "dumped", "mirrored", "unbacked", "cached", "dataless")
	facts := backupFactsFrom(entries, holders("ace", "g14"))
	proceed, err := stepBackups(t.Context(), p, facts, backupT0)
	if err != nil || proceed {
		t.Fatalf("proceed=%v err=%v; the plan must wait for the backups it asked", proceed, err)
	}
	want := "ace/copied ace/dumped g14/copied"
	if got := strings.Join(seat.asked, " "); got != want {
		t.Fatalf("asked %q, want %q", got, want)
	}
	if p.Backups == nil || p.Backups.State != inventory.BackupPending || len(p.Backups.Asked) != 3 {
		t.Fatalf("the plan's record: %+v", p.Backups)
	}
}

// A machine with no backup holder is asked nothing, the plan says so, and it does not hold the plan.
func TestANodeWithNoBackupHolderAsksNothingAndThePlanSaysSo(t *testing.T) {
	seat := &aFakeBackupSeat{}
	seat.install(t)
	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "bare"}, anItem("a", ""))}
	p := aPlanOf("copied")
	proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0)
	if err != nil || proceed {
		t.Fatalf("proceed=%v err=%v", proceed, err)
	}
	if strings.Join(seat.asked, " ") != "ace/copied" {
		t.Fatalf("asked %v", seat.asked)
	}
	if p.Backups == nil || len(p.Backups.NoHolder) != 1 || p.Backups.NoHolder[0] != "bare" {
		t.Fatalf("the plan must say bare has no holder: %+v", p.Backups)
	}
	// Only the no-holder machine: nothing to wait for, the plan goes on, and still says it.
	only := []inventory.Entry{aModuleWith(t, "copied", []string{"bare"}, anItem("a", ""))}
	q := aPlanOf("copied")
	proceed, err = stepBackups(t.Context(), q, backupFactsFrom(only, holders("ace")), backupT0)
	if err != nil || !proceed || q.Backups == nil || q.Backups.NoHolder[0] != "bare" || q.Backups.State != inventory.BackupDone {
		t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, q.Backups)
	}
}

// Nothing of the plan goes on until every backup answered done; a backup is not asked twice.
func TestNothingIsAskedOfTheBuildOrSentUntilEveryBackupAnsweredDone(t *testing.T) {
	seat := &aFakeBackupSeat{nights: map[string]*night{}}
	seat.install(t)
	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))}
	facts := backupFactsFrom(entries, holders("ace", "g14"))
	p := aPlanOf("copied")
	if proceed, _ := stepBackups(t.Context(), p, facts, backupT0); proceed {
		t.Fatal("went on before any backup answered")
	}
	// ace is done, g14 still running (its last night is from before the ask).
	seat.nights["ace/copied"] = &night{OK: true, At: backupT0.Add(time.Minute)}
	seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(-time.Hour)}
	if proceed, _ := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed {
		t.Fatal("went on while g14's backup had not answered")
	}
	seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(3 * time.Minute)}
	proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(4*time.Minute))
	if err != nil || !proceed || p.Backups.State != inventory.BackupDone {
		t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, p.Backups)
	}
	if len(seat.asked) != 2 {
		t.Fatalf("asked %v; each backup is asked once", seat.asked)
	}
}

// A failed backup, and a silent one past the bound, are recorded with the backup's own words and the plan stops.
func TestAFailedOrSilentBackupIsRecordedAndThePlanStopsAsking(t *testing.T) {
	seat := &aFakeBackupSeat{nights: map[string]*night{}}
	seat.install(t)
	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))}
	facts := backupFactsFrom(entries, holders("ace", "g14"))
	p := aPlanOf("copied")
	stepBackups(t.Context(), p, facts, backupT0)
	seat.nights["ace/copied"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"}
	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin+time.Minute)); proceed || err != nil {
		t.Fatalf("proceed=%v err=%v", proceed, err)
	}
	got := map[string]inventory.PlanBackup{}
	for _, b := range p.Backups.Asked {
		got[b.Machine] = b
	}
	if got["ace"].State != inventory.BackupFailed || got["ace"].Words != "no space left on device" {
		t.Fatalf("ace: %+v", got["ace"])
	}
	if got["g14"].State != inventory.BackupFailed || !strings.Contains(got["g14"].Words, "did not answer in ten minutes") {
		t.Fatalf("g14: %+v", got["g14"])
	}
	if p.Backups.State != inventory.BackupFailed {
		t.Fatalf("state %q", p.Backups.State)
	}
	// It stays stopped, asks nothing more, and says no retry in the plan's note.
	n := len(seat.asked)
	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(time.Hour)); proceed || err != nil || len(seat.asked) != n {
		t.Fatalf("a failed backup must stay stopped: proceed=%v err=%v asked=%v", proceed, err, seat.asked)
	}
}

// A holder that refuses the ask says why, in the record.
func TestAHolderThatRefusesTheAskIsRecordedFailedWithItsWords(t *testing.T) {
	seat := &aFakeBackupSeat{refuse: map[string]string{"ace/copied": "copied declares nothing to back up"}}
	seat.install(t)
	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace"}, anItem("a", ""))}
	p := aPlanOf("copied")
	if proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0); proceed || err != nil {
		t.Fatalf("proceed=%v err=%v", proceed, err)
	}
	if b := p.Backups.Asked[0]; b.State != inventory.BackupFailed || b.Words != "copied declares nothing to back up" {
		t.Fatalf("%+v", b)
	}
}

// A plan moving no backed-up item asks nothing and keeps no record (design 47: costs nothing).
func TestAPlanThatMovesNoBackedUpItemAsksNothing(t *testing.T) {
	seat := &aFakeBackupSeat{}
	seat.install(t)
	entries := []inventory.Entry{
		aModuleWith(t, "dataless", []string{"ace"}),
		aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)),
	}
	p := aPlanOf("dataless", "mirrored")
	proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0)
	if err != nil || !proceed || p.Backups != nil || len(seat.asked) != 0 {
		t.Fatalf("proceed=%v err=%v record=%+v asked=%v", proceed, err, p.Backups, seat.asked)
	}
}

// The holders of a node seat are every machine's, and a source that fails is no machine without a holder.
func TestTheBackupHoldersAreEveryMachinesAndAFailedReadIsNotNoHolder(t *testing.T) {
	entries := func(context.Context) ([]inventory.Entry, error) {
		return []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))}, nil
	}
	f, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return holders("ace", "g14"), nil }, t.Context())
	if err != nil || !f.holder["ace"] || !f.holder["g14"] {
		t.Fatalf("%+v %v", f, err)
	}
	if _, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return nil, fmt.Errorf("store down") },
		t.Context()); err == nil || !strings.Contains(err.Error(), "store down") {
		t.Fatalf("a failed read of the holders must fail the step: %v", err)
	}
}