mesh-catalog/modules/restic/cmd/restic-backups/live_integration_test.go

//go:build integration

package main

// The module's live path on a bus of this test alone (design 47 G §561, §563, A-150, A-155): the night's own broker account.

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
	"git.novox.be/novox/mesh-sdk/go/process"
	"git.novox.be/novox/mesh-sdk/go/testbus"
)

// A put-off is a key of the state put-off with a time to live of its own: it is listed while it lasts, and the bus removing it
// at the end of its time is heard as the expiry, once; a key released by the holder is heard as nothing.
func TestAPutOffKeyIsListedAndItsExpiryIsHeardFromTheBus(t *testing.T) {
	url := testbus.URL(t)
	nc, err := nats.Connect(url)
	if err != nil {
		t.Fatal(err)
	}
	defer nc.Close()
	js, _ := jetstream.New(nc)
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}); err != nil {
		t.Fatal(err)
	}
	conn, err := process.Connect(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"})
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	st := stateOfPutOffs{conn.State(PutOffStateName)}
	now := time.Date(2026, 11, 12, 3, 0, 0, 0, time.UTC)
	expired := make(chan string, 4)
	if _, err := st.Hold("postgres.store", PutOff{Restore: "act-7", Since: now}, 2*time.Second, func() { expired <- "store" }); err != nil {
		t.Fatal(err)
	}
	release, err := st.Hold("postgres.dumps", PutOff{Restore: "act-7", Since: now}, time.Hour, func() { expired <- "dumps" })
	if err != nil {
		t.Fatal(err)
	}
	all, err := st.All()
	if err != nil || len(all) != 2 || all["postgres.store"].Restore != "act-7" || !all["postgres.store"].Since.Equal(now) {
		t.Fatalf("the keys %+v, %v", all, err)
	}
	select {
	case who := <-expired:
		if who != "store" {
			t.Fatalf("%s expired", who)
		}
	case <-time.After(30 * time.Second):
		t.Fatal("the bus did not remove the key at the end of its time")
	}
	release()
	for {
		all, err = st.All()
		if err != nil {
			t.Fatal(err)
		}
		if len(all) == 0 {
			break
		}
		select {
		case who := <-expired:
			t.Fatalf("%s expired after its release", who)
		case <-time.After(100 * time.Millisecond):
		}
	}
}

// The night as the node starts it: its credential is read from where the manifest puts it (the process night's MESH_BROKER_FILE,
// the module's own secret broker), it says each item started and ended on the seat node-backup, gives way to a restore of the
// recovery module that runs, puts its items off as keys of the state put-off and starts again when the restore says restored; a
// restore that does not end within put-off-bound expires its keys on the bus, writes backup-waits into the module's findings and
// ends the wait. The streams are the ones the bus composer makes.
func TestTheNightSaysItsItemsAndGivesWayToARestoreOnTheBus(t *testing.T) {
	// The manifest's own place of the credential, and the environment the night is given.
	raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
	if err != nil {
		t.Fatal(err)
	}
	var m struct {
		OwnSecrets map[string]string `json:"own-secrets"`
		Resources  []struct {
			ID  string            `json:"id"`
			Env map[string]string `json:"env"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatal(err)
	}
	dir := t.TempDir()
	env := map[string]string{}
	for _, r := range m.Resources {
		if r.ID == "night" {
			for k, v := range r.Env {
				env[k] = strings.ReplaceAll(v, "${dir:state}", dir)
			}
		}
	}
	if m.OwnSecrets["broker"] != "${dir:state}/broker" || env[process.BrokerFileEnv] != filepath.Join(dir, "broker") {
		t.Fatalf("the manifest places the broker file at %q and gives the night %q", m.OwnSecrets["broker"], env[process.BrokerFileEnv])
	}
	url := testbus.URL(t)
	cred, _ := json.Marshal(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"})
	if err := os.WriteFile(env[process.BrokerFileEnv], cred, 0o600); err != nil {
		t.Fatal(err)
	}
	live, err := connectItems(func(k string) string { return env[k] })
	if err != nil || live == nil {
		t.Fatalf("the night's credential at its place was not used: %v", err)
	}
	defer live.Close()

	nc, err := nats.Connect(url)
	if err != nil {
		t.Fatal(err)
	}
	defer nc.Close()
	js, _ := jetstream.New(nc)
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()
	for _, b := range []jetstream.KeyValueConfig{{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}, {Bucket: "restic_findings"}} {
		if _, err := js.CreateKeyValue(ctx, b); err != nil {
			t.Fatal(err)
		}
	}
	for stream, subject := range map[string]string{
		streamItemStarted: "mesh.seat.node-backup.event.item-started.>", streamItemEnded: "mesh.seat.node-backup.event.item-ended.>",
		streamRestoreStarted: "mesh.seat.mesh-recovery.event.restore-started.>", streamRestored: "mesh.seat.mesh-recovery.event.restored.>"} {
		if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil {
			t.Fatal(err)
		}
	}
	for _, s := range []jetstream.StreamConfig{{Name: "RESTIC_FINDINGS", Subjects: []string{"mesh.stream.restic.event.>"}},
		{Name: "RESTIC_FINDING_EVENTS", Subjects: []string{"mesh.bus.restic.finding-events.>"}},
		{Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}} {
		if _, err := js.CreateStream(ctx, s); err != nil {
			t.Fatal(err)
		}
	}
	// The assigner's answer to who holds the seat mesh-broker: the module postgres.
	if _, err := nc.Subscribe("mesh.seat.mesh-module-assigner.call.assignments.*", func(msg *nats.Msg) {
		body, _ := json.Marshal(map[string]any{"result": map[string]any{"seats": []any{map[string]any{"seat": BrokerSeat,
			"holders": []any{map[string]any{"module": "postgres"}}}}}, "node": "ace"})
		_ = msg.Respond(body)
	}); err != nil {
		t.Fatal(err)
	}
	// The seat mesh-broker's show names the module's `finding-events` entry (A-274).
	if _, err := nc.Subscribe("mesh.seat.mesh-broker.call.show.*", func(msg *nats.Msg) {
		body, _ := json.Marshal(map[string]any{"result": map[string]any{"binding": map[string]any{"local": "finding-events", "storage": "stream",
			"template": "events", "stream": "RESTIC_FINDING_EVENTS", "subject": "mesh.bus.restic.finding-events"}}, "node": "ace"})
		_ = msg.Respond(body)
	}); err != nil {
		t.Fatal(err)
	}
	findings, closeFindings, err := live.Findings(ctx, t.Logf)
	if err != nil {
		t.Fatal(err)
	}
	defer closeFindings()

	b := &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: time.Now, Say: quiet, Bus: live,
		BrokerModule: live.Broker, PutOffs: live.PutOffs(), Findings: findings,
		Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }}
	say := func(event, act string) {
		body, _ := json.Marshal(map[string]any{"act": act, "at": time.Now().UTC()})
		if _, err := js.Publish(ctx, "mesh.seat.mesh-recovery.event."+event+".postgres.store", body); err != nil {
			t.Fatal(err)
		}
	}
	lastEnded := func(item string) string {
		msg, err := js.Conn().RequestWithContext(ctx, "$JS.API.STREAM.MSG.GET."+streamItemEnded, []byte(`{"last_by_subj":"`+
			itemEventSubject(EventItemEnded, "ace", "postgres", item)+`"}`))
		if err != nil {
			t.Fatal(err)
		}
		var got struct {
			Message struct {
				Data []byte `json:"data"`
			} `json:"message"`
		}
		if err := json.Unmarshal(msg.Data, &got); err != nil {
			t.Fatal(err)
		}
		return string(got.Message.Data)
	}

	// A restore runs and ends: the items are put off as keys, then start again and end done.
	say("restore-started", "act-7")
	kv, _ := js.KeyValue(ctx, "restic_"+PutOffStateName)
	watch, err := kv.Watch(ctx, "postgres.store", jetstream.UpdatesOnly())
	if err != nil {
		t.Fatal(err)
	}
	ended := make(chan Night, 1)
	go func() { ended <- b.one(ctx, declaredOf(t, b, "postgres"), false) }()
	select {
	case e := <-watch.Updates():
		if e == nil || e.Operation() != jetstream.KeyValuePut {
			t.Fatalf("the put-off key came as %+v", e)
		}
	case <-time.After(30 * time.Second):
		t.Fatal("no put-off key was written")
	}
	say("restored", "act-7")
	if n := <-ended; !n.OK {
		t.Fatalf("the night after the restore: %+v", n)
	}
	left, err := live.Unended()
	if err != nil || len(left) != 0 {
		t.Fatalf("items left started: %+v, %v", left, err)
	}
	if got := lastEnded("store"); !strings.Contains(got, `"outcome":"done"`) {
		t.Fatalf("the last end of the item store: %s", got)
	}

	// A restore that does not end: the bound (a second) passes, the bus removes the keys, backup-waits is written.
	b.PutOffBound = time.Second
	say("restore-started", "act-8")
	began := time.Now()
	n := b.one(ctx, declaredOf(t, b, "postgres"), false)
	if n.OK || !strings.Contains(n.Error, "put-off-bound") {
		t.Fatalf("the night past the bound: %+v", n)
	}
	if took := time.Since(began); took > 20*time.Second {
		t.Fatalf("the wait ended %v after it began: the bound's expiry did not end it", took)
	}
	fkv, _ := js.KeyValue(ctx, "restic_findings")
	ks, err := fkv.Keys(ctx)
	waits := 0
	for _, k := range ks {
		if strings.HasSuffix(k, "."+FindingBackupWaits) && strings.HasPrefix(k, "ace.postgres=2e") {
			waits++
		}
	}
	if err != nil || waits != 1 {
		t.Fatalf("the findings kept: %v, %v", ks, err)
	}
	// A-274: the finding events went under the finding-events binding, none on the old stream.
	for stream, want := range map[string]bool{"RESTIC_FINDING_EVENTS": true, "RESTIC_FINDINGS": false} {
		st, err := js.Stream(ctx, stream)
		if err != nil {
			t.Fatal(err)
		}
		if info, err := st.Info(ctx); err != nil || (info.State.Msgs > 0) != want {
			t.Fatalf("%s holds %d finding events (%v), want some: %v", stream, info.State.Msgs, err, want)
		}
	}
}