mesh-catalog/modules/restic/cmd/restic-backups/items_test.go

package main

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)

// itemBusFake records every call, in order, as the backup holder makes them.
type itemBusFake struct {
	log        []string
	restoring  []string // what each read of the recovery module's restores answers, in turn
	unended    []ItemRef
	runFailure error
	// waiting runs while the holder waits for a restore's restored; hold makes the wait last until its context ends.
	waiting func()
	hold    bool
}

func (f *itemBusFake) Say(event string, ref ItemRef, body map[string]any) error {
	line := event + " " + ref.Module + "." + ref.Item
	if o, ok := body["outcome"]; ok {
		line += " " + fmt.Sprint(o)
	}
	if r, ok := body["restore"]; ok {
		line += " restore=" + fmt.Sprint(r)
	}
	if e, ok := body["error"]; ok {
		line += " error=" + fmt.Sprint(e)
	}
	if _, ok := body["at"]; !ok {
		line += " NO-AT"
	}
	f.log = append(f.log, line)
	return nil
}
func (f *itemBusFake) Unended() ([]ItemRef, error) { return f.unended, nil }
func (f *itemBusFake) Restoring(module string) (string, error) {
	f.log = append(f.log, "read restores of "+module)
	if len(f.restoring) == 0 {
		return "", nil
	}
	act := f.restoring[0]
	f.restoring = f.restoring[1:]
	return act, nil
}
func (f *itemBusFake) WaitRestored(ctx context.Context, act string) error {
	f.log = append(f.log, "restored "+act)
	if f.waiting != nil {
		f.waiting()
	}
	if f.hold {
		<-ctx.Done()
		return ctx.Err()
	}
	return nil
}

// putOffFake is the state put-off: its keys with the time to live each was written with; expire plays the bus removing one.
type putOffFake struct {
	keys    map[string]PutOff
	ttl     map[string]time.Duration
	expired map[string]func()
}

func newPutOffFake() *putOffFake {
	return &putOffFake{keys: map[string]PutOff{}, ttl: map[string]time.Duration{}, expired: map[string]func(){}}
}

func (f *putOffFake) Hold(key string, rec PutOff, ttl time.Duration, expired func()) (func(), error) {
	f.keys[key], f.ttl[key], f.expired[key] = rec, ttl, expired
	return func() { delete(f.keys, key); delete(f.expired, key) }, nil
}
func (f *putOffFake) All() (map[string]PutOff, error) { return f.keys, nil }
func (f *putOffFake) expire(key string) {
	cb := f.expired[key]
	delete(f.keys, key)
	delete(f.expired, key)
	cb()
}

// findingsFake holds the findings written and not cleared.
type findingsFake struct {
	held map[string]meshconditions.Finding
}

func (f *findingsFake) Write(_ context.Context, x meshconditions.Finding) error {
	f.held[x.Scope+"."+x.ID+"."+x.Kind] = x
	return nil
}
func (f *findingsFake) Clear(_ context.Context, scope, id, kind, _ string) error {
	delete(f.held, scope+"."+id+"."+kind)
	return nil
}
func (f *findingsFake) Held() []meshconditions.Finding {
	var out []meshconditions.Finding
	for _, x := range f.held {
		out = append(out, x)
	}
	return out
}

func itemsBackups(t *testing.T, bus *itemBusFake, broker string, runErr error) *Backups {
	t.Helper()
	return &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: func() time.Time { return time.Date(2026, 11, 12, 3, 0, 0, 0, time.UTC) },
		Say: quiet, Bus: bus, BrokerModule: func() (string, error) { return broker, nil },
		Run: func(_ context.Context, name string, args ...string) (string, error) {
			if name == "restic" && len(args) > 5 && args[5] == "backup" {
				return "", runErr
			}
			return "[]", nil
		}}
}

func declaredOf(t *testing.T, b *Backups, module string) Declared {
	t.Helper()
	all, err := b.Declared()
	must(t, err)
	for _, d := range all {
		if d.Module == module {
			return d
		}
	}
	t.Fatalf("%s is not declared", module)
	return Declared{}
}

// Design 47 G §563 (BKP4): every item of a backed-up module says item-started, then item-ended with its outcome and `at`; a
// cache item says nothing; a failed backup ends its items failed with why; the holder reads no restore for a module that does
// not hold the seat mesh-broker.
func TestEveryItemOfABackedUpModuleSaysItsStartAndItsEnd(t *testing.T) {
	bus := &itemBusFake{}
	b := itemsBackups(t, bus, "somebody-else", nil)
	if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); !n.OK {
		t.Fatalf("%+v", n)
	}
	want := []string{"item-started postgres.store", "item-started postgres.dumps", "item-ended postgres.store done", "item-ended postgres.dumps done"}
	if !reflect.DeepEqual(bus.log, want) {
		t.Fatalf("said %q", bus.log)
	}
	bus.log = nil
	b = itemsBackups(t, bus, "somebody-else", errors.New("disk full"))
	if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); n.OK {
		t.Fatal("a failed backup was ok")
	}
	if len(bus.log) != 4 || bus.log[2] != "item-ended postgres.store failed error=disk full" {
		t.Fatalf("said %q", bus.log)
	}
}

// Design 47 G §563 (BKP5): for the holder of mesh-broker the holder says its start, then reads the recovery module's restores;
// a restore that runs puts the items off naming its act, waits for that act's restored, starts again, and reads once more.
func TestTheBackupOfTheBusGivesWayToARestoreThatRuns(t *testing.T) {
	bus := &itemBusFake{restoring: []string{"act-7"}}
	b := itemsBackups(t, bus, "postgres", nil)
	state := newPutOffFake()
	b.PutOffs, b.PutOffBound = state, 45*time.Minute
	// While it waits, `backed-up` lists the item put off with the restore's act and since when (BKP8, §302, §664).
	bus.waiting = func() {
		got, err := b.BackedUp(context.Background(), "postgres")
		must(t, err)
		if rec, ok := state.keys["postgres.store"]; !ok || rec.Restore != "act-7" || state.ttl["postgres.store"] != 45*time.Minute {
			t.Errorf("the put-off is not a key of the state put-off with the bound as its time to live: %+v %v", state.keys, state.ttl)
		}
		var store *ItemReport
		for i := range got[0].Data {
			if got[0].Data[i].ID == "store" {
				store = &got[0].Data[i]
			}
		}
		if store == nil || store.PutOff == nil || store.PutOff.Restore != "act-7" || !store.PutOff.Since.Equal(b.Now()) {
			t.Errorf("backed-up says the item as %+v", store)
		}
	}
	if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); !n.OK {
		t.Fatalf("%+v", n)
	}
	want := []string{
		"item-started postgres.store", "item-started postgres.dumps", "read restores of postgres",
		"item-ended postgres.store put-off restore=act-7", "item-ended postgres.dumps put-off restore=act-7",
		"restored act-7",
		"item-started postgres.store", "item-started postgres.dumps", "read restores of postgres",
		"item-ended postgres.store done", "item-ended postgres.dumps done"}
	if !reflect.DeepEqual(bus.log, want) {
		t.Fatalf("said %q", bus.log)
	}
	got, err := b.BackedUp(context.Background(), "postgres")
	must(t, err)
	for _, it := range got[0].Data {
		if it.PutOff != nil {
			t.Errorf("%s is still put off once it ran: %+v", it.ID, it.PutOff)
		}
	}
}

// Design 47 G §561, §563, A-155 (BKP6): a wait for a restore past put-off-bound, the key of the state put-off expiring, writes
// the finding backup-waits for the item (scope the node), ends the wait (the backup fails, naming the bound) and holds the night's
// lock no longer; the finding is cleared when the item starts again.
func TestAWaitPastItsBoundWritesBackupWaitsAndEndsTheWait(t *testing.T) {
	bus := &itemBusFake{restoring: []string{"act-7"}, hold: true}
	b := itemsBackups(t, bus, "postgres", nil)
	state, findings := newPutOffFake(), &findingsFake{held: map[string]meshconditions.Finding{}}
	b.PutOffs, b.Findings = state, findings
	bus.waiting = func() { state.expire("postgres.dumps") }
	n := b.one(context.Background(), declaredOf(t, b, "postgres"), false)
	if n.OK || !strings.Contains(n.Error, "put-off-bound") || !strings.Contains(n.Error, "act-7") {
		t.Fatalf("the night %+v", n)
	}
	f, ok := findings.held["ace.postgres=2edumps.backup-waits"]
	if !ok || len(findings.held) != 1 || f.Scope != "ace" || f.Kind != FindingBackupWaits || !strings.Contains(f.Summary, "act-7") {
		t.Fatalf("the findings held: %+v", findings.held)
	}
	if len(state.keys) != 0 {
		t.Errorf("the keys left: %+v", state.keys)
	}
	// The next night: the item starts again and the finding is cleared.
	bus.restoring, bus.hold, bus.waiting = nil, false, nil
	if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); !n.OK || len(findings.held) != 0 {
		t.Fatalf("the night %+v, findings %+v", n, findings.held)
	}
}

// Design 47 G §561 (A-155): the setting put-off-bound is minutes in the file the manifest places; sixty when none is given.
func TestPutOffBoundReadsItsSettingFile(t *testing.T) {
	file := filepath.Join(t.TempDir(), "put-off-bound")
	for text, want := range map[string]time.Duration{"15\n": 15 * time.Minute, "": DefaultPutOffBound} {
		must(t, os.WriteFile(file, []byte(text), 0o600))
		got, err := putOffBoundFrom(func(string) string { return file })
		if err != nil || got != want {
			t.Errorf("%q gave %v, %v", text, got, err)
		}
	}
	if got, _ := putOffBoundFrom(func(string) string { return "" }); got != DefaultPutOffBound {
		t.Errorf("no file gave %v", got)
	}
	must(t, os.WriteFile(file, []byte("soon"), 0o600))
	if _, err := putOffBoundFrom(func(string) string { return file }); err == nil {
		t.Error("a bound that is no number was taken")
	}
}

// Design 47 G §563 (BKP4): at its start the holder ends every item it said started and never ended, failed with the error stopped.
func TestItemsLeftStartedAreEndedFailedStoppedAtTheStart(t *testing.T) {
	bus := &itemBusFake{unended: []ItemRef{{"postgres", "store"}}}
	b := itemsBackups(t, bus, "postgres", nil)
	must(t, b.StopStale())
	if !reflect.DeepEqual(bus.log, []string{"item-ended postgres.store failed error=stopped"}) {
		t.Fatalf("said %q", bus.log)
	}
}