jschoubben
ee56d56922
An integration test starts the night as the node does: its credential is read from where the manifest places it (the process night's MESH_BROKER_FILE, the module's own-secrets broker), it says each item started and ended on the seat node-backup, gives way to a restore that runs (items put off as keys of the state put-off, started again on the restored) and, past put-off-bound, the bus's expiry writes backup-waits into the findings. It found that WaitRestored held a fetch of a minute after its context ended, so an expiry ended the wait up to a minute late: it now consumes the restored events and ends on the context. Builds: design 47 G §563, §561, §554, A-150, A-155 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
245 lines
8.8 KiB
Go
245 lines
8.8 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
// The module's live path on a bus of this test alone (design 47 G §561, §563, A-150, A-155): the night's own broker account.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/testbus"
|
|
)
|
|
|
|
// A put-off is a key of the state put-off with a time to live of its own: it is listed while it lasts, and the bus removing it
|
|
// at the end of its time is heard as the expiry, once; a key released by the holder is heard as nothing.
|
|
func TestAPutOffKeyIsListedAndItsExpiryIsHeardFromTheBus(t *testing.T) {
|
|
url := testbus.URL(t)
|
|
nc, err := nats.Connect(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer nc.Close()
|
|
js, _ := jetstream.New(nc)
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conn, err := process.Connect(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
st := stateOfPutOffs{conn.State(PutOffStateName)}
|
|
now := time.Date(2026, 11, 12, 3, 0, 0, 0, time.UTC)
|
|
expired := make(chan string, 4)
|
|
if _, err := st.Hold("postgres.store", PutOff{Restore: "act-7", Since: now}, 2*time.Second, func() { expired <- "store" }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
release, err := st.Hold("postgres.dumps", PutOff{Restore: "act-7", Since: now}, time.Hour, func() { expired <- "dumps" })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
all, err := st.All()
|
|
if err != nil || len(all) != 2 || all["postgres.store"].Restore != "act-7" || !all["postgres.store"].Since.Equal(now) {
|
|
t.Fatalf("the keys %+v, %v", all, err)
|
|
}
|
|
select {
|
|
case who := <-expired:
|
|
if who != "store" {
|
|
t.Fatalf("%s expired", who)
|
|
}
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatal("the bus did not remove the key at the end of its time")
|
|
}
|
|
release()
|
|
for {
|
|
all, err = st.All()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(all) == 0 {
|
|
break
|
|
}
|
|
select {
|
|
case who := <-expired:
|
|
t.Fatalf("%s expired after its release", who)
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
|
|
// The night as the node starts it: its credential is read from where the manifest puts it (the process night's MESH_BROKER_FILE,
|
|
// the module's own secret broker), it says each item started and ended on the seat node-backup, gives way to a restore of the
|
|
// recovery module that runs, puts its items off as keys of the state put-off and starts again when the restore says restored; a
|
|
// restore that does not end within put-off-bound expires its keys on the bus, writes backup-waits into the module's findings and
|
|
// ends the wait. The streams are the ones the bus composer makes.
|
|
func TestTheNightSaysItsItemsAndGivesWayToARestoreOnTheBus(t *testing.T) {
|
|
// The manifest's own place of the credential, and the environment the night is given.
|
|
raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var m struct {
|
|
OwnSecrets map[string]string `json:"own-secrets"`
|
|
Resources []struct {
|
|
ID string `json:"id"`
|
|
Env map[string]string `json:"env"`
|
|
} `json:"resources"`
|
|
}
|
|
if err := json.Unmarshal(raw, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dir := t.TempDir()
|
|
env := map[string]string{}
|
|
for _, r := range m.Resources {
|
|
if r.ID == "night" {
|
|
for k, v := range r.Env {
|
|
env[k] = strings.ReplaceAll(v, "${dir:state}", dir)
|
|
}
|
|
}
|
|
}
|
|
if m.OwnSecrets["broker"] != "${dir:state}/broker" || env[process.BrokerFileEnv] != filepath.Join(dir, "broker") {
|
|
t.Fatalf("the manifest places the broker file at %q and gives the night %q", m.OwnSecrets["broker"], env[process.BrokerFileEnv])
|
|
}
|
|
url := testbus.URL(t)
|
|
cred, _ := json.Marshal(process.Account{URL: url, Node: "ace", Module: "restic", User: "ace.restic", Password: "unused"})
|
|
if err := os.WriteFile(env[process.BrokerFileEnv], cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
live, err := connectItems(func(k string) string { return env[k] })
|
|
if err != nil || live == nil {
|
|
t.Fatalf("the night's credential at its place was not used: %v", err)
|
|
}
|
|
defer live.Close()
|
|
|
|
nc, err := nats.Connect(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer nc.Close()
|
|
js, _ := jetstream.New(nc)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancel()
|
|
for _, b := range []jetstream.KeyValueConfig{{Bucket: "restic_" + PutOffStateName, LimitMarkerTTL: time.Second}, {Bucket: "restic_findings"}} {
|
|
if _, err := js.CreateKeyValue(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for stream, subject := range map[string]string{
|
|
streamItemStarted: "mesh.seat.node-backup.event.item-started.>", streamItemEnded: "mesh.seat.node-backup.event.item-ended.>",
|
|
streamRestoreStarted: "mesh.seat.mesh-recovery.event.restore-started.>", streamRestored: "mesh.seat.mesh-recovery.event.restored.>"} {
|
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, s := range []jetstream.StreamConfig{{Name: "RESTIC_FINDINGS", Subjects: []string{"mesh.stream.restic.event.>"}},
|
|
{Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}} {
|
|
if _, err := js.CreateStream(ctx, s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// The assigner's answer to who holds the seat mesh-broker: the module postgres.
|
|
if _, err := nc.Subscribe("mesh.seat.mesh-module-assigner.call.assignments.*", func(msg *nats.Msg) {
|
|
body, _ := json.Marshal(map[string]any{"result": map[string]any{"seats": []any{map[string]any{"seat": BrokerSeat,
|
|
"holders": []any{map[string]any{"module": "postgres"}}}}}, "node": "ace"})
|
|
_ = msg.Respond(body)
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
findings, closeFindings, err := live.Findings(ctx, t.Logf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer closeFindings()
|
|
|
|
b := &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: time.Now, Say: quiet, Bus: live,
|
|
BrokerModule: live.Broker, PutOffs: live.PutOffs(), Findings: findings,
|
|
Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }}
|
|
say := func(event, act string) {
|
|
body, _ := json.Marshal(map[string]any{"act": act, "at": time.Now().UTC()})
|
|
if _, err := js.Publish(ctx, "mesh.seat.mesh-recovery.event."+event+".postgres.store", body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
lastEnded := func(item string) string {
|
|
msg, err := js.Conn().RequestWithContext(ctx, "$JS.API.STREAM.MSG.GET."+streamItemEnded, []byte(`{"last_by_subj":"`+
|
|
itemEventSubject(EventItemEnded, "ace", "postgres", item)+`"}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got struct {
|
|
Message struct {
|
|
Data []byte `json:"data"`
|
|
} `json:"message"`
|
|
}
|
|
if err := json.Unmarshal(msg.Data, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(got.Message.Data)
|
|
}
|
|
|
|
// A restore runs and ends: the items are put off as keys, then start again and end done.
|
|
say("restore-started", "act-7")
|
|
kv, _ := js.KeyValue(ctx, "restic_"+PutOffStateName)
|
|
watch, err := kv.Watch(ctx, "postgres.store", jetstream.UpdatesOnly())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ended := make(chan Night, 1)
|
|
go func() { ended <- b.one(ctx, declaredOf(t, b, "postgres"), false) }()
|
|
select {
|
|
case e := <-watch.Updates():
|
|
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
|
t.Fatalf("the put-off key came as %+v", e)
|
|
}
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatal("no put-off key was written")
|
|
}
|
|
say("restored", "act-7")
|
|
if n := <-ended; !n.OK {
|
|
t.Fatalf("the night after the restore: %+v", n)
|
|
}
|
|
left, err := live.Unended()
|
|
if err != nil || len(left) != 0 {
|
|
t.Fatalf("items left started: %+v, %v", left, err)
|
|
}
|
|
if got := lastEnded("store"); !strings.Contains(got, `"outcome":"done"`) {
|
|
t.Fatalf("the last end of the item store: %s", got)
|
|
}
|
|
|
|
// A restore that does not end: the bound (a second) passes, the bus removes the keys, backup-waits is written.
|
|
b.PutOffBound = time.Second
|
|
say("restore-started", "act-8")
|
|
began := time.Now()
|
|
n := b.one(ctx, declaredOf(t, b, "postgres"), false)
|
|
if n.OK || !strings.Contains(n.Error, "put-off-bound") {
|
|
t.Fatalf("the night past the bound: %+v", n)
|
|
}
|
|
if took := time.Since(began); took > 20*time.Second {
|
|
t.Fatalf("the wait ended %v after it began: the bound's expiry did not end it", took)
|
|
}
|
|
fkv, _ := js.KeyValue(ctx, "restic_findings")
|
|
ks, err := fkv.Keys(ctx)
|
|
waits := 0
|
|
for _, k := range ks {
|
|
if strings.HasSuffix(k, "."+FindingBackupWaits) && strings.HasPrefix(k, "ace.postgres=2e") {
|
|
waits++
|
|
}
|
|
}
|
|
if err != nil || waits != 1 {
|
|
t.Fatalf("the findings kept: %v, %v", ks, err)
|
|
}
|
|
}
|