BKP4-BKP6 live path: restic's item events over the bus with the night's own broker account; the wait for a restore ends when the bound's expiry cancels it

An integration test starts the night as the node does: its credential is read from where the manifest places it (the process
night's MESH_BROKER_FILE, the module's own-secrets broker), it says each item started and ended on the seat node-backup, gives
way to a restore that runs (items put off as keys of the state put-off, started again on the restored) and, past put-off-bound,
the bus's expiry writes backup-waits into the findings. It found that WaitRestored held a fetch of a minute after its context
ended, so an expiry ended the wait up to a minute late: it now consumes the restored events and ends on the context.

Builds: design 47 G §563, §561, §554, A-150, A-155

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h

modules/restic/cmd/restic-backups/itembus.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -219,20 +220,26 @@ func (l *liveBus) WaitRestored(ctx context.Context, want string) error {
 	if err != nil {
 		return err
 	}
-	for {
-		m, err := cons.Next(jetstream.FetchMaxWait(time.Minute))
-		if err != nil {
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
-			continue
-		}
+	done := make(chan struct{})
+	var once sync.Once
+	cc, err := cons.Consume(func(m jetstream.Msg) {
 		var b struct {
 			Act string `json:"act"`
 		}
 		if json.Unmarshal(m.Data(), &b) == nil && b.Act == want {
-			return nil
+			once.Do(func() { close(done) })
 		}
+	})
+	if err != nil {
+		return err
+	}
+	defer cc.Stop()
+	// The wait ends on the restored or when ctx ends (the bound's expiry cancels it), never on a fetch's own time.
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
 	}
 }
 

modules/restic/cmd/restic-backups/live_integration_test.go

@@ -6,12 +6,17 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 
+	"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
 	"git.novox.be/novox/mesh-sdk/go/process"
 	"git.novox.be/novox/mesh-sdk/go/testbus"
 )
@@ -74,3 +79,166 @@ func TestAPutOffKeyIsListedAndItsExpiryIsHeardFromTheBus(t *testing.T) {
 		}
 	}
 }
+
+// The night as the node starts it: its credential is read from where the manifest puts it (the process night's MESH_BROKER_FILE,
+// the module's own secret broker), it says each item started and ended on the seat node-backup, gives way to a restore of the
+// recovery module that runs, puts its items off as keys of the state put-off and starts again when the restore says restored; a
+// restore that does not end within put-off-bound expires its keys on the bus, writes backup-waits into the module's findings and
+// ends the wait. The streams are the ones the bus composer makes.
+func TestTheNightSaysItsItemsAndGivesWayToARestoreOnTheBus(t *testing.T) {
+	// The manifest's own place of the credential, and the environment the night is given.
+	raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var m struct {
+		OwnSecrets map[string]string `json:"own-secrets"`
+		Resources  []struct {
+			ID  string            `json:"id"`
+			Env map[string]string `json:"env"`
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(raw, &m); err != nil {
+		t.Fatal(err)
+	}
+	dir := t.TempDir()
+	env := map[string]string{}
+	for _, r := range m.Resources {
+		if r.ID == "night" {
+			for k, v := range r.Env {
+				env[k] = strings.ReplaceAll(v, "${dir:state}", dir)
+			}
+		}
+	}
+	if m.OwnSecrets["broker"] != "${dir:state}/broker" || env[process.BrokerFileEnv] != filepath.Join(dir, "broker") {
+		t.Fatalf("the manifest places the broker file at %q and gives the night %q", m.OwnSecrets["broker"], env[process.BrokerFileEnv])
+	}
+	url := testbus.URL(t)
+	cred, _ := json.Marshal(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"})
+	if err := os.WriteFile(env[process.BrokerFileEnv], cred, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	live, err := connectItems(func(k string) string { return env[k] })
+	if err != nil || live == nil {
+		t.Fatalf("the night's credential at its place was not used: %v", err)
+	}
+	defer live.Close()
+
+	nc, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nc.Close()
+	js, _ := jetstream.New(nc)
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	for _, b := range []jetstream.KeyValueConfig{{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}, {Bucket: "restic_findings"}} {
+		if _, err := js.CreateKeyValue(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for stream, subject := range map[string]string{
+		streamItemStarted: "mesh.seat.node-backup.event.item-started.>", streamItemEnded: "mesh.seat.node-backup.event.item-ended.>",
+		streamRestoreStarted: "mesh.seat.mesh-recovery.event.restore-started.>", streamRestored: "mesh.seat.mesh-recovery.event.restored.>"} {
+		if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, s := range []jetstream.StreamConfig{{Name: "RESTIC_FINDINGS", Subjects: []string{"mesh.stream.restic.event.>"}},
+		{Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}} {
+		if _, err := js.CreateStream(ctx, s); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// The assigner's answer to who holds the seat mesh-broker: the module postgres.
+	if _, err := nc.Subscribe("mesh.seat.mesh-module-assigner.call.assignments.*", func(msg *nats.Msg) {
+		body, _ := json.Marshal(map[string]any{"result": map[string]any{"seats": []any{map[string]any{"seat": BrokerSeat,
+			"holders": []any{map[string]any{"module": "postgres"}}}}}, "node": "ace"})
+		_ = msg.Respond(body)
+	}); err != nil {
+		t.Fatal(err)
+	}
+	findings, closeFindings, err := live.Findings(ctx, t.Logf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeFindings()
+
+	b := &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: time.Now, Say: quiet, Bus: live,
+		BrokerModule: live.Broker, PutOffs: live.PutOffs(), Findings: findings,
+		Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }}
+	say := func(event, act string) {
+		body, _ := json.Marshal(map[string]any{"act": act, "at": time.Now().UTC()})
+		if _, err := js.Publish(ctx, "mesh.seat.mesh-recovery.event."+event+".postgres.store", body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	lastEnded := func(item string) string {
+		msg, err := js.Conn().RequestWithContext(ctx, "$JS.API.STREAM.MSG.GET."+streamItemEnded, []byte(`{"last_by_subj":"`+
+			itemEventSubject(EventItemEnded, "ace", "postgres", item)+`"}`))
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got struct {
+			Message struct {
+				Data []byte `json:"data"`
+			} `json:"message"`
+		}
+		if err := json.Unmarshal(msg.Data, &got); err != nil {
+			t.Fatal(err)
+		}
+		return string(got.Message.Data)
+	}
+
+	// A restore runs and ends: the items are put off as keys, then start again and end done.
+	say("restore-started", "act-7")
+	kv, _ := js.KeyValue(ctx, "restic_"+PutOffStateName)
+	watch, err := kv.Watch(ctx, "postgres.store", jetstream.UpdatesOnly())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ended := make(chan Night, 1)
+	go func() { ended <- b.one(ctx, declaredOf(t, b, "postgres"), false) }()
+	select {
+	case e := <-watch.Updates():
+		if e == nil || e.Operation() != jetstream.KeyValuePut {
+			t.Fatalf("the put-off key came as %+v", e)
+		}
+	case <-time.After(30 * time.Second):
+		t.Fatal("no put-off key was written")
+	}
+	say("restored", "act-7")
+	if n := <-ended; !n.OK {
+		t.Fatalf("the night after the restore: %+v", n)
+	}
+	left, err := live.Unended()
+	if err != nil || len(left) != 0 {
+		t.Fatalf("items left started: %+v, %v", left, err)
+	}
+	if got := lastEnded("store"); !strings.Contains(got, `"outcome":"done"`) {
+		t.Fatalf("the last end of the item store: %s", got)
+	}
+
+	// A restore that does not end: the bound (a second) passes, the bus removes the keys, backup-waits is written.
+	b.PutOffBound = time.Second
+	say("restore-started", "act-8")
+	began := time.Now()
+	n := b.one(ctx, declaredOf(t, b, "postgres"), false)
+	if n.OK || !strings.Contains(n.Error, "put-off-bound") {
+		t.Fatalf("the night past the bound: %+v", n)
+	}
+	if took := time.Since(began); took > 20*time.Second {
+		t.Fatalf("the wait ended %v after it began: the bound's expiry did not end it", took)
+	}
+	fkv, _ := js.KeyValue(ctx, "restic_findings")
+	ks, err := fkv.Keys(ctx)
+	waits := 0
+	for _, k := range ks {
+		if strings.HasSuffix(k, "."+FindingBackupWaits) && strings.HasPrefix(k, "ace.postgres=2e") {
+			waits++
+		}
+	}
+	if err != nil || waits != 1 {
+		t.Fatalf("the findings kept: %v, %v", ks, err)
+	}
+}