diff --git a/modules/restic/cmd/restic-backups/itembus.go b/modules/restic/cmd/restic-backups/itembus.go index 316a4b071..cd0d95a41 100644 --- a/modules/restic/cmd/restic-backups/itembus.go +++ b/modules/restic/cmd/restic-backups/itembus.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "strings" + "sync" "sync/atomic" "time" @@ -219,20 +220,26 @@ func (l *liveBus) WaitRestored(ctx context.Context, want string) error { if err != nil { return err } - for { - m, err := cons.Next(jetstream.FetchMaxWait(time.Minute)) - if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - continue - } + done := make(chan struct{}) + var once sync.Once + cc, err := cons.Consume(func(m jetstream.Msg) { var b struct { Act string `json:"act"` } if json.Unmarshal(m.Data(), &b) == nil && b.Act == want { - return nil + once.Do(func() { close(done) }) } + }) + if err != nil { + return err + } + defer cc.Stop() + // The wait ends on the restored or when ctx ends (the bound's expiry cancels it), never on a fetch's own time. + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() } } diff --git a/modules/restic/cmd/restic-backups/live_integration_test.go b/modules/restic/cmd/restic-backups/live_integration_test.go index 6044481e2..87d16daae 100644 --- a/modules/restic/cmd/restic-backups/live_integration_test.go +++ b/modules/restic/cmd/restic-backups/live_integration_test.go @@ -6,12 +6,17 @@ package main import ( "context" + "encoding/json" + "os" + "path/filepath" + "strings" "testing" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/testbus" ) @@ -74,3 +79,166 @@ func TestAPutOffKeyIsListedAndItsExpiryIsHeardFromTheBus(t *testing.T) { } } } + +// The night as the node starts it: its credential is read from where the manifest puts it (the process night's MESH_BROKER_FILE, +// the module's own secret broker), it says each item started and ended on the seat node-backup, gives way to a restore of the +// recovery module that runs, puts its items off as keys of the state put-off and starts again when the restore says restored; a +// restore that does not end within put-off-bound expires its keys on the bus, writes backup-waits into the module's findings and +// ends the wait. The streams are the ones the bus composer makes. +func TestTheNightSaysItsItemsAndGivesWayToARestoreOnTheBus(t *testing.T) { + // The manifest's own place of the credential, and the environment the night is given. + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + OwnSecrets map[string]string `json:"own-secrets"` + Resources []struct { + ID string `json:"id"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + env := map[string]string{} + for _, r := range m.Resources { + if r.ID == "night" { + for k, v := range r.Env { + env[k] = strings.ReplaceAll(v, "${dir:state}", dir) + } + } + } + if m.OwnSecrets["broker"] != "${dir:state}/broker" || env[process.BrokerFileEnv] != filepath.Join(dir, "broker") { + t.Fatalf("the manifest places the broker file at %q and gives the night %q", m.OwnSecrets["broker"], env[process.BrokerFileEnv]) + } + url := testbus.URL(t) + cred, _ := json.Marshal(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"}) + if err := os.WriteFile(env[process.BrokerFileEnv], cred, 0o600); err != nil { + t.Fatal(err) + } + live, err := connectItems(func(k string) string { return env[k] }) + if err != nil || live == nil { + t.Fatalf("the night's credential at its place was not used: %v", err) + } + defer live.Close() + + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + for _, b := range []jetstream.KeyValueConfig{{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}, {Bucket: "restic_findings"}} { + if _, err := js.CreateKeyValue(ctx, b); err != nil { + t.Fatal(err) + } + } + for stream, subject := range map[string]string{ + streamItemStarted: "mesh.seat.node-backup.event.item-started.>", streamItemEnded: "mesh.seat.node-backup.event.item-ended.>", + streamRestoreStarted: "mesh.seat.mesh-recovery.event.restore-started.>", streamRestored: "mesh.seat.mesh-recovery.event.restored.>"} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + } + for _, s := range []jetstream.StreamConfig{{Name: "RESTIC_FINDINGS", Subjects: []string{"mesh.stream.restic.event.>"}}, + {Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}} { + if _, err := js.CreateStream(ctx, s); err != nil { + t.Fatal(err) + } + } + // The assigner's answer to who holds the seat mesh-broker: the module postgres. + if _, err := nc.Subscribe("mesh.seat.mesh-module-assigner.call.assignments.*", func(msg *nats.Msg) { + body, _ := json.Marshal(map[string]any{"result": map[string]any{"seats": []any{map[string]any{"seat": BrokerSeat, + "holders": []any{map[string]any{"module": "postgres"}}}}}, "node": "ace"}) + _ = msg.Respond(body) + }); err != nil { + t.Fatal(err) + } + findings, closeFindings, err := live.Findings(ctx, t.Logf) + if err != nil { + t.Fatal(err) + } + defer closeFindings() + + b := &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: time.Now, Say: quiet, Bus: live, + BrokerModule: live.Broker, PutOffs: live.PutOffs(), Findings: findings, + Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }} + say := func(event, act string) { + body, _ := json.Marshal(map[string]any{"act": act, "at": time.Now().UTC()}) + if _, err := js.Publish(ctx, "mesh.seat.mesh-recovery.event."+event+".postgres.store", body); err != nil { + t.Fatal(err) + } + } + lastEnded := func(item string) string { + msg, err := js.Conn().RequestWithContext(ctx, "$JS.API.STREAM.MSG.GET."+streamItemEnded, []byte(`{"last_by_subj":"`+ + itemEventSubject(EventItemEnded, "ace", "postgres", item)+`"}`)) + if err != nil { + t.Fatal(err) + } + var got struct { + Message struct { + Data []byte `json:"data"` + } `json:"message"` + } + if err := json.Unmarshal(msg.Data, &got); err != nil { + t.Fatal(err) + } + return string(got.Message.Data) + } + + // A restore runs and ends: the items are put off as keys, then start again and end done. + say("restore-started", "act-7") + kv, _ := js.KeyValue(ctx, "restic_"+PutOffStateName) + watch, err := kv.Watch(ctx, "postgres.store", jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + ended := make(chan Night, 1) + go func() { ended <- b.one(ctx, declaredOf(t, b, "postgres"), false) }() + select { + case e := <-watch.Updates(): + if e == nil || e.Operation() != jetstream.KeyValuePut { + t.Fatalf("the put-off key came as %+v", e) + } + case <-time.After(30 * time.Second): + t.Fatal("no put-off key was written") + } + say("restored", "act-7") + if n := <-ended; !n.OK { + t.Fatalf("the night after the restore: %+v", n) + } + left, err := live.Unended() + if err != nil || len(left) != 0 { + t.Fatalf("items left started: %+v, %v", left, err) + } + if got := lastEnded("store"); !strings.Contains(got, `"outcome":"done"`) { + t.Fatalf("the last end of the item store: %s", got) + } + + // A restore that does not end: the bound (a second) passes, the bus removes the keys, backup-waits is written. + b.PutOffBound = time.Second + say("restore-started", "act-8") + began := time.Now() + n := b.one(ctx, declaredOf(t, b, "postgres"), false) + if n.OK || !strings.Contains(n.Error, "put-off-bound") { + t.Fatalf("the night past the bound: %+v", n) + } + if took := time.Since(began); took > 20*time.Second { + t.Fatalf("the wait ended %v after it began: the bound's expiry did not end it", took) + } + fkv, _ := js.KeyValue(ctx, "restic_findings") + ks, err := fkv.Keys(ctx) + waits := 0 + for _, k := range ks { + if strings.HasSuffix(k, "."+FindingBackupWaits) && strings.HasPrefix(k, "ace.postgres=2e") { + waits++ + } + } + if err != nil || waits != 1 { + t.Fatalf("the findings kept: %v, %v", ks, err) + } +}