//go:build integration package main // The module's live path on a bus of this test alone (design 47 G §561, §563, A-150, A-155): the night's own broker account. import ( "context" "encoding/json" "os" "path/filepath" "strings" "testing" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/testbus" ) // A put-off is a key of the state put-off with a time to live of its own: it is listed while it lasts, and the bus removing it // at the end of its time is heard as the expiry, once; a key released by the holder is heard as nothing. func TestAPutOffKeyIsListedAndItsExpiryIsHeardFromTheBus(t *testing.T) { url := testbus.URL(t) nc, err := nats.Connect(url) if err != nil { t.Fatal(err) } defer nc.Close() js, _ := jetstream.New(nc) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}); err != nil { t.Fatal(err) } conn, err := process.Connect(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"}) if err != nil { t.Fatal(err) } defer conn.Close() st := stateOfPutOffs{conn.State(PutOffStateName)} now := time.Date(2026, 11, 12, 3, 0, 0, 0, time.UTC) expired := make(chan string, 4) if _, err := st.Hold("postgres.store", PutOff{Restore: "act-7", Since: now}, 2*time.Second, func() { expired <- "store" }); err != nil { t.Fatal(err) } release, err := st.Hold("postgres.dumps", PutOff{Restore: "act-7", Since: now}, time.Hour, func() { expired <- "dumps" }) if err != nil { t.Fatal(err) } all, err := st.All() if err != nil || len(all) != 2 || all["postgres.store"].Restore != "act-7" || !all["postgres.store"].Since.Equal(now) { t.Fatalf("the keys %+v, %v", all, err) } select { case who := <-expired: if who != "store" { t.Fatalf("%s expired", who) } case <-time.After(30 * time.Second): t.Fatal("the bus did not remove the key at the end of its time") } release() for { all, err = st.All() if err != nil { t.Fatal(err) } if len(all) == 0 { break } select { case who := <-expired: t.Fatalf("%s expired after its release", who) case <-time.After(100 * time.Millisecond): } } } // The night as the node starts it: its credential is read from where the manifest puts it (the process night's MESH_BROKER_FILE, // the module's own secret broker), it says each item started and ended on the seat node-backup, gives way to a restore of the // recovery module that runs, puts its items off as keys of the state put-off and starts again when the restore says restored; a // restore that does not end within put-off-bound expires its keys on the bus, writes backup-waits into the module's findings and // ends the wait. The streams are the ones the bus composer makes. func TestTheNightSaysItsItemsAndGivesWayToARestoreOnTheBus(t *testing.T) { // The manifest's own place of the credential, and the environment the night is given. raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) if err != nil { t.Fatal(err) } var m struct { OwnSecrets map[string]string `json:"own-secrets"` Resources []struct { ID string `json:"id"` Env map[string]string `json:"env"` } `json:"resources"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } dir := t.TempDir() env := map[string]string{} for _, r := range m.Resources { if r.ID == "night" { for k, v := range r.Env { env[k] = strings.ReplaceAll(v, "${dir:state}", dir) } } } if m.OwnSecrets["broker"] != "${dir:state}/broker" || env[process.BrokerFileEnv] != filepath.Join(dir, "broker") { t.Fatalf("the manifest places the broker file at %q and gives the night %q", m.OwnSecrets["broker"], env[process.BrokerFileEnv]) } url := testbus.URL(t) cred, _ := json.Marshal(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"}) if err := os.WriteFile(env[process.BrokerFileEnv], cred, 0o600); err != nil { t.Fatal(err) } live, err := connectItems(func(k string) string { return env[k] }) if err != nil || live == nil { t.Fatalf("the night's credential at its place was not used: %v", err) } defer live.Close() nc, err := nats.Connect(url) if err != nil { t.Fatal(err) } defer nc.Close() js, _ := jetstream.New(nc) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() for _, b := range []jetstream.KeyValueConfig{{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}, {Bucket: "restic_findings"}} { if _, err := js.CreateKeyValue(ctx, b); err != nil { t.Fatal(err) } } for stream, subject := range map[string]string{ streamItemStarted: "mesh.seat.node-backup.event.item-started.>", streamItemEnded: "mesh.seat.node-backup.event.item-ended.>", streamRestoreStarted: "mesh.seat.mesh-recovery.event.restore-started.>", streamRestored: "mesh.seat.mesh-recovery.event.restored.>"} { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil { t.Fatal(err) } } for _, s := range []jetstream.StreamConfig{{Name: "RESTIC_FINDINGS", Subjects: []string{"mesh.stream.restic.event.>"}}, {Name: "RESTIC_FINDING_EVENTS", Subjects: []string{"mesh.bus.restic.finding-events.>"}}, {Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}} { if _, err := js.CreateStream(ctx, s); err != nil { t.Fatal(err) } } // The assigner's answer to who holds the seat mesh-broker: the module postgres. if _, err := nc.Subscribe("mesh.seat.mesh-module-assigner.call.assignments.*", func(msg *nats.Msg) { body, _ := json.Marshal(map[string]any{"result": map[string]any{"seats": []any{map[string]any{"seat": BrokerSeat, "holders": []any{map[string]any{"module": "postgres"}}}}}, "node": "ace"}) _ = msg.Respond(body) }); err != nil { t.Fatal(err) } // The seat mesh-broker's show names the module's `finding-events` entry (A-274). if _, err := nc.Subscribe("mesh.seat.mesh-broker.call.show.*", func(msg *nats.Msg) { body, _ := json.Marshal(map[string]any{"result": map[string]any{"binding": map[string]any{"local": "finding-events", "storage": "stream", "template": "events", "stream": "RESTIC_FINDING_EVENTS", "subject": "mesh.bus.restic.finding-events"}}, "node": "ace"}) _ = msg.Respond(body) }); err != nil { t.Fatal(err) } findings, closeFindings, err := live.Findings(ctx, t.Logf) if err != nil { t.Fatal(err) } defer closeFindings() b := &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: time.Now, Say: quiet, Bus: live, BrokerModule: live.Broker, PutOffs: live.PutOffs(), Findings: findings, Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }} say := func(event, act string) { body, _ := json.Marshal(map[string]any{"act": act, "at": time.Now().UTC()}) if _, err := js.Publish(ctx, "mesh.seat.mesh-recovery.event."+event+".postgres.store", body); err != nil { t.Fatal(err) } } lastEnded := func(item string) string { msg, err := js.Conn().RequestWithContext(ctx, "$JS.API.STREAM.MSG.GET."+streamItemEnded, []byte(`{"last_by_subj":"`+ itemEventSubject(EventItemEnded, "ace", "postgres", item)+`"}`)) if err != nil { t.Fatal(err) } var got struct { Message struct { Data []byte `json:"data"` } `json:"message"` } if err := json.Unmarshal(msg.Data, &got); err != nil { t.Fatal(err) } return string(got.Message.Data) } // A restore runs and ends: the items are put off as keys, then start again and end done. say("restore-started", "act-7") kv, _ := js.KeyValue(ctx, "restic_"+PutOffStateName) watch, err := kv.Watch(ctx, "postgres.store", jetstream.UpdatesOnly()) if err != nil { t.Fatal(err) } ended := make(chan Night, 1) go func() { ended <- b.one(ctx, declaredOf(t, b, "postgres"), false) }() select { case e := <-watch.Updates(): if e == nil || e.Operation() != jetstream.KeyValuePut { t.Fatalf("the put-off key came as %+v", e) } case <-time.After(30 * time.Second): t.Fatal("no put-off key was written") } say("restored", "act-7") if n := <-ended; !n.OK { t.Fatalf("the night after the restore: %+v", n) } left, err := live.Unended() if err != nil || len(left) != 0 { t.Fatalf("items left started: %+v, %v", left, err) } if got := lastEnded("store"); !strings.Contains(got, `"outcome":"done"`) { t.Fatalf("the last end of the item store: %s", got) } // A restore that does not end: the bound (a second) passes, the bus removes the keys, backup-waits is written. b.PutOffBound = time.Second say("restore-started", "act-8") began := time.Now() n := b.one(ctx, declaredOf(t, b, "postgres"), false) if n.OK || !strings.Contains(n.Error, "put-off-bound") { t.Fatalf("the night past the bound: %+v", n) } if took := time.Since(began); took > 20*time.Second { t.Fatalf("the wait ended %v after it began: the bound's expiry did not end it", took) } fkv, _ := js.KeyValue(ctx, "restic_findings") ks, err := fkv.Keys(ctx) waits := 0 for _, k := range ks { if strings.HasSuffix(k, "."+FindingBackupWaits) && strings.HasPrefix(k, "ace.postgres=2e") { waits++ } } if err != nil || waits != 1 { t.Fatalf("the findings kept: %v, %v", ks, err) } // A-274: the finding events went under the finding-events binding, none on the old stream. for stream, want := range map[string]bool{"RESTIC_FINDING_EVENTS": true, "RESTIC_FINDINGS": false} { st, err := js.Stream(ctx, stream) if err != nil { t.Fatal(err) } if info, err := st.Info(ctx); err != nil || (info.State.Msgs > 0) != want { t.Fatalf("%s holds %d finding events (%v), want some: %v", stream, info.State.Msgs, err, want) } } }