package keeper import ( "archive/tar" "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" "github.com/klauspost/compress/s2" "nats-tools/internal/busarchive" ) // tarBus is the fake bus whose snapshot is a real archive of the streams asked for, as the server's is. type tarBus struct { *fakeBus dir string n int } func (b *tarBus) Snapshot(ctx context.Context, streams []string) (string, error) { if _, err := b.fakeBus.Snapshot(ctx, streams); err != nil { return "", err } b.n++ path := filepath.Join(b.dir, fmt.Sprintf(".entry-%d.tar", b.n)) return path, os.WriteFile(path, archiveOf(streams), 0o600) } func sha(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } // archiveOf is a snapshot archive of the streams, each holding one block. func archiveOf(streams []string) []byte { var out bytes.Buffer tw := tar.NewWriter(&out) put := func(name string, b []byte) { _ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))}) _, _ = tw.Write(b) } m := busarchive.Manifest{Format: 1, Finished: time.Date(2026, 11, 9, 3, 0, 0, 0, time.UTC)} files := map[string][]byte{} for _, name := range streams { var raw bytes.Buffer itw := tar.NewWriter(&raw) _ = itw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: 3}) _, _ = itw.Write([]byte("abc")) itw.Close() var arc bytes.Buffer w := s2.NewWriter(&arc) _, _ = w.Write(raw.Bytes()) w.Close() meta := []byte(`{"config":{"name":"` + name + `"}}`) e := busarchive.StreamEntry{Name: name, Meta: "streams/" + name + "/backup.json", MetaSHA256: sha(meta), Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(arc.Len()), ArchiveSHA256: sha(arc.Bytes())} m.Streams = append(m.Streams, e) files[e.Meta], files[e.Archive] = meta, arc.Bytes() } raw, _ := json.Marshal(m) put(busarchive.ManifestName, raw) for _, s := range m.Streams { put(s.Meta, files[s.Meta]) put(s.Archive, files[s.Archive]) } tw.Close() return out.Bytes() } // keptBus is the fake bus's streams and consumers, as the server holds them. type keptBus struct { streams map[string]bool positions []Position set []Position restored [][]string refuse error } func (b *keptBus) Existing(_ context.Context, names []string) ([]string, error) { var out []string for _, n := range names { if b.streams[n] { out = append(out, n) } } return out, nil } func (b *keptBus) Positions(context.Context, string) ([]Position, error) { return b.positions, nil } func (b *keptBus) SetPositions(_ context.Context, ps []Position) error { b.set = append(b.set, ps...) return nil } func (b *keptBus) RestoreArchive(_ context.Context, a *busarchive.Archive) error { if err := b.refuse; err != nil { b.refuse = nil // refused once, as a server that was busy return err } var names []string for _, s := range a.Manifest.Streams { if b.streams[s.Name] { return fmt.Errorf("the server already holds %s", s.Name) } names = append(names, s.Name) } for _, n := range names { b.streams[n] = true } b.restored = append(b.restored, names) return nil } func (b *keptBus) DeleteStreams(_ context.Context, names []string) error { for _, n := range names { delete(b.streams, n) } return nil } func keptKeeper(t *testing.T) (*Keeper, *keptBus, *memStore, *[]string) { t.Helper() k, fb, store, _ := newKeeper(FirstView{}) dir := t.TempDir() kb := &keptBus{streams: map[string]bool{"KV_idx_modules": true}, positions: []Position{{Stream: "forge", Consumer: "ace_idx", AckFloor: 41, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}, {Stream: "KV_idx_modules", Consumer: "ace_idx", AckFloor: 7, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}}} acts := &[]string{} k.Bus, k.Kept, k.KeptDir = &tarBus{fakeBus: fb, dir: dir}, kb, dir k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil } return k, kb, store, acts } func take(t *testing.T, k *Keeper) *Request { t.Helper() r := &Request{Verb: VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Caller: "ace.restic", Why: "before form 1"} if _, err := k.SnapshotKept(context.Background(), r); err != nil { t.Fatal(err) } return r } // A module's kept data on the bus has a restore point before its module moves, and a restore never goes over the live // data (design 47 "A module's kept data on the bus, backed up and restored with its delivery"). func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) { for _, row := range []struct { name string run func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) }{ {"a restore point holds the archive, the positions and the catch-up tool, keyed by form and fingerprint", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { r := take(t, k) want := filepath.Join(k.KeptDir, "points", "idx", "modules", "f0-abc123") if r.Dir != want { t.Fatalf("restore point in %s, want %s", r.Dir, want) } raw, err := os.ReadFile(filepath.Join(want, PointFile)) if err != nil { t.Fatal(err) } var p Point if err := json.Unmarshal(raw, &p); err != nil || p.CatchUp != "indexer_take_in" || len(p.Positions) != 2 || p.Streams[0] != "KV_idx_modules" || p.Bytes == 0 { t.Errorf("point.json = %s (%v)", raw, err) } if _, err := os.Stat(filepath.Join(want, ArchiveFile)); err != nil { t.Error(err) } if _, _, found, _ := store.Get(StateKept, PointKey("idx", "modules", 0, "abc123")); !found { t.Error("the restore point is not recorded in the state kept") } if len(*acts) != 1 || !strings.Contains((*acts)[0], "took the restore point") { t.Errorf("hand acts %v", *acts) } }}, {"aside-only sets the live data aside whole and restores nothing", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", AsideOnly: true, Caller: "ace.restic", Why: "newer build stopped"} if _, err := k.RestoreKept(context.Background(), r); err != nil { t.Fatal(err) } if kb.streams["KV_idx_modules"] || len(kb.restored) != 0 { t.Errorf("streams %v restored %v: the live data is removed and nothing restored", kb.streams, kb.restored) } aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside")) if err != nil || len(aside) != 1 || r.Dir == "" { t.Fatalf("aside: %v %v dir %q", aside, err, r.Dir) } }}, {"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"} _, err := k.SnapshotKept(context.Background(), r) if err == nil || !strings.Contains(err.Error(), "KV_idx_gone") { t.Fatalf("err = %v", err) } if _, err := os.Stat(filepath.Join(k.KeptDir, "points")); err == nil { t.Error("a directory was made for a snapshot that was not taken") } }}, {"a name that is not one plain name is refused", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { for _, r := range []*Request{{Module: "../x", Local: "modules", Fingerprint: "a"}, {Module: "idx", Local: "mod/ules", Fingerprint: "a"}, {Module: "idx", Local: "modules", Fingerprint: ""}} { if _, err := k.SnapshotKept(context.Background(), r); err == nil { t.Errorf("%+v was taken", r) } } }}, {"a restore sets the live data aside whole, removes it, restores the archive and sets back the other consumers", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { p := take(t, k) r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: p.Dir, Caller: "ace.restic", Why: "revert"} if _, err := k.RestoreKept(context.Background(), r); err != nil { t.Fatal(err) } if !kb.streams["KV_idx_modules"] || len(kb.restored) != 1 { t.Errorf("streams %v restored %v", kb.streams, kb.restored) } if r.CatchUp != "indexer_take_in" { t.Errorf("the catch-up tool is %q", r.CatchUp) } aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside")) if err != nil || len(aside) != 1 { t.Fatalf("aside: %v %v", aside, err) } if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", aside[0].Name(), ArchiveFile)); err != nil { t.Error("the live data's archive is not kept") } if _, _, found, _ := store.Get(StateKept, "aside."+aside[0].Name()); !found { t.Error("what is set aside is not recorded") } // Only the consumer on a stream the point did not carry is made again; a stream's own comes with its snapshot. if len(kb.set) != 1 || kb.set[0].Stream != "forge" || kb.set[0].AckFloor != 41 { t.Errorf("positions set back: %+v", kb.set) } }}, {"a restore point of another fingerprint or a linked file is refused before anything is removed", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { p := take(t, k) bad := *p bad.Fingerprint = "other" if _, err := k.RestoreKept(context.Background(), &bad); err == nil || !kb.streams["KV_idx_modules"] { t.Errorf("err = %v, live %v", err, kb.streams) } link := filepath.Join(p.Dir, ArchiveFile) real := filepath.Join(p.Dir, "real.tar") if err := os.Rename(link, real); err != nil { t.Fatal(err) } if err := os.Symlink(real, link); err != nil { t.Fatal(err) } r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} if _, err := k.RestoreKept(context.Background(), r); err == nil || !strings.Contains(err.Error(), "plain file") || !kb.streams["KV_idx_modules"] { t.Errorf("err = %v, live %v", err, kb.streams) } }}, {"a live data that cannot be set aside stays where it is", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { p := take(t, k) k.Bus.(*tarBus).fakeBus.refuse["snapshot"] = errors.New("the server refused") r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} if _, err := k.RestoreKept(context.Background(), r); err == nil || !kb.streams["KV_idx_modules"] || !strings.Contains(err.Error(), "nothing was removed") { t.Errorf("err = %v, live %v", err, kb.streams) } }}, {"a restore that fails puts the live data back and says so", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { p := take(t, k) kb.refuse = errors.New("the server refused the archive") r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} _, err := k.RestoreKept(context.Background(), r) if err == nil || !strings.Contains(err.Error(), "the live data is back as it was") { t.Fatalf("err = %v", err) } }}, {"set-aside data goes only on a call that says why and is recorded first", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { p := take(t, k) if _, err := k.RestoreKept(context.Background(), &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}); err != nil { t.Fatal(err) } entries, _ := os.ReadDir(filepath.Join(k.KeptDir, "aside")) id := entries[0].Name() if _, err := k.DiscardAside(&Request{From: id}); err == nil { t.Error("discarded with no why") } if _, err := k.DiscardAside(&Request{From: "../points", Why: "x"}); err == nil { t.Error("discarded a path") } k.Act = func(string, string, string) error { return errors.New("the log is down") } r := &Request{From: id, Why: "the revert held", Caller: "ace.operator"} k.record(r, "x") // sets Unrecorded as a failing log does if _, err := k.DiscardAside(r); err == nil { t.Error("discarded with the act unrecorded") } if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err != nil { t.Error("the data went with its act unrecorded") } k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil } if _, err := k.DiscardAside(&Request{From: id, Why: "the revert held", Caller: "ace.operator"}); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err == nil { t.Error("still there") } if _, _, found, _ := store.Get(StateKept, "aside."+id); found { t.Error("its record stays") } }}, } { t.Run(row.name, func(t *testing.T) { k, kb, store, acts := keptKeeper(t) row.run(t, k, kb, store, acts) }) } }