jschoubben
b6884796e7
Verbs snapshot-kept and restore-kept (with aside-only and discard-aside) on the seat mesh-broker, handed to the keeper; taken over from the join's 0480e16f and c4e88eaa, without the verbs aside and discard-aside and the tool nats_kept, which no design line gives. Builds: design 47 G §296, §297, §302, §337, §405 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
312 lines
12 KiB
Go
312 lines
12 KiB
Go
package keeper
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/klauspost/compress/s2"
|
|
|
|
"nats-tools/internal/busarchive"
|
|
)
|
|
|
|
// tarBus is the fake bus whose snapshot is a real archive of the streams asked for, as the server's is.
|
|
type tarBus struct {
|
|
*fakeBus
|
|
dir string
|
|
n int
|
|
}
|
|
|
|
func (b *tarBus) Snapshot(ctx context.Context, streams []string) (string, error) {
|
|
if _, err := b.fakeBus.Snapshot(ctx, streams); err != nil {
|
|
return "", err
|
|
}
|
|
b.n++
|
|
path := filepath.Join(b.dir, fmt.Sprintf(".entry-%d.tar", b.n))
|
|
return path, os.WriteFile(path, archiveOf(streams), 0o600)
|
|
}
|
|
|
|
func sha(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
|
|
|
// archiveOf is a snapshot archive of the streams, each holding one block.
|
|
func archiveOf(streams []string) []byte {
|
|
var out bytes.Buffer
|
|
tw := tar.NewWriter(&out)
|
|
put := func(name string, b []byte) {
|
|
_ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))})
|
|
_, _ = tw.Write(b)
|
|
}
|
|
m := busarchive.Manifest{Format: 1, Finished: time.Date(2026, 11, 9, 3, 0, 0, 0, time.UTC)}
|
|
files := map[string][]byte{}
|
|
for _, name := range streams {
|
|
var raw bytes.Buffer
|
|
itw := tar.NewWriter(&raw)
|
|
_ = itw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: 3})
|
|
_, _ = itw.Write([]byte("abc"))
|
|
itw.Close()
|
|
var arc bytes.Buffer
|
|
w := s2.NewWriter(&arc)
|
|
_, _ = w.Write(raw.Bytes())
|
|
w.Close()
|
|
meta := []byte(`{"config":{"name":"` + name + `"}}`)
|
|
e := busarchive.StreamEntry{Name: name, Meta: "streams/" + name + "/backup.json", MetaSHA256: sha(meta),
|
|
Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(arc.Len()), ArchiveSHA256: sha(arc.Bytes())}
|
|
m.Streams = append(m.Streams, e)
|
|
files[e.Meta], files[e.Archive] = meta, arc.Bytes()
|
|
}
|
|
raw, _ := json.Marshal(m)
|
|
put(busarchive.ManifestName, raw)
|
|
for _, s := range m.Streams {
|
|
put(s.Meta, files[s.Meta])
|
|
put(s.Archive, files[s.Archive])
|
|
}
|
|
tw.Close()
|
|
return out.Bytes()
|
|
}
|
|
|
|
// keptBus is the fake bus's streams and consumers, as the server holds them.
|
|
type keptBus struct {
|
|
streams map[string]bool
|
|
positions []Position
|
|
set []Position
|
|
restored [][]string
|
|
refuse error
|
|
}
|
|
|
|
func (b *keptBus) Existing(_ context.Context, names []string) ([]string, error) {
|
|
var out []string
|
|
for _, n := range names {
|
|
if b.streams[n] {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
func (b *keptBus) Positions(context.Context, string) ([]Position, error) { return b.positions, nil }
|
|
func (b *keptBus) SetPositions(_ context.Context, ps []Position) error {
|
|
b.set = append(b.set, ps...)
|
|
return nil
|
|
}
|
|
func (b *keptBus) RestoreArchive(_ context.Context, a *busarchive.Archive) error {
|
|
if err := b.refuse; err != nil {
|
|
b.refuse = nil // refused once, as a server that was busy
|
|
return err
|
|
}
|
|
var names []string
|
|
for _, s := range a.Manifest.Streams {
|
|
if b.streams[s.Name] {
|
|
return fmt.Errorf("the server already holds %s", s.Name)
|
|
}
|
|
names = append(names, s.Name)
|
|
}
|
|
for _, n := range names {
|
|
b.streams[n] = true
|
|
}
|
|
b.restored = append(b.restored, names)
|
|
return nil
|
|
}
|
|
func (b *keptBus) DeleteStreams(_ context.Context, names []string) error {
|
|
for _, n := range names {
|
|
delete(b.streams, n)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func keptKeeper(t *testing.T) (*Keeper, *keptBus, *memStore, *[]string) {
|
|
t.Helper()
|
|
k, fb, store, _ := newKeeper(FirstView{})
|
|
dir := t.TempDir()
|
|
kb := &keptBus{streams: map[string]bool{"KV_idx_modules": true},
|
|
positions: []Position{{Stream: "forge", Consumer: "ace_idx", AckFloor: 41, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)},
|
|
{Stream: "KV_idx_modules", Consumer: "ace_idx", AckFloor: 7, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}}}
|
|
acts := &[]string{}
|
|
k.Bus, k.Kept, k.KeptDir = &tarBus{fakeBus: fb, dir: dir}, kb, dir
|
|
k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil }
|
|
return k, kb, store, acts
|
|
}
|
|
|
|
func take(t *testing.T, k *Keeper) *Request {
|
|
t.Helper()
|
|
r := &Request{Verb: VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Caller: "ace.restic", Why: "before form 1"}
|
|
if _, err := k.SnapshotKept(context.Background(), r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// A module's kept data on the bus has a restore point before its module moves, and a restore never goes over the live
|
|
// data (design 47 "A module's kept data on the bus, backed up and restored with its delivery").
|
|
func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) {
|
|
for _, row := range []struct {
|
|
name string
|
|
run func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string)
|
|
}{
|
|
{"a restore point holds the archive, the positions and the catch-up tool, keyed by form and fingerprint", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
r := take(t, k)
|
|
want := filepath.Join(k.KeptDir, "points", "idx", "modules", "f0-abc123")
|
|
if r.Dir != want {
|
|
t.Fatalf("restore point in %s, want %s", r.Dir, want)
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(want, PointFile))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var p Point
|
|
if err := json.Unmarshal(raw, &p); err != nil || p.CatchUp != "indexer_take_in" || len(p.Positions) != 2 || p.Streams[0] != "KV_idx_modules" || p.Bytes == 0 {
|
|
t.Errorf("point.json = %s (%v)", raw, err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(want, ArchiveFile)); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if _, _, found, _ := store.Get(StateKept, PointKey("idx", "modules", 0, "abc123")); !found {
|
|
t.Error("the restore point is not recorded in the state kept")
|
|
}
|
|
if len(*acts) != 1 || !strings.Contains((*acts)[0], "took the restore point") {
|
|
t.Errorf("hand acts %v", *acts)
|
|
}
|
|
}},
|
|
{"aside-only sets the live data aside whole and restores nothing", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", AsideOnly: true, Caller: "ace.restic", Why: "newer build stopped"}
|
|
if _, err := k.RestoreKept(context.Background(), r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if kb.streams["KV_idx_modules"] || len(kb.restored) != 0 {
|
|
t.Errorf("streams %v restored %v: the live data is removed and nothing restored", kb.streams, kb.restored)
|
|
}
|
|
aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
|
|
if err != nil || len(aside) != 1 || r.Dir == "" {
|
|
t.Fatalf("aside: %v %v dir %q", aside, err, r.Dir)
|
|
}
|
|
}},
|
|
{"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"}
|
|
_, err := k.SnapshotKept(context.Background(), r)
|
|
if err == nil || !strings.Contains(err.Error(), "KV_idx_gone") {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(k.KeptDir, "points")); err == nil {
|
|
t.Error("a directory was made for a snapshot that was not taken")
|
|
}
|
|
}},
|
|
{"a name that is not one plain name is refused", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
for _, r := range []*Request{{Module: "../x", Local: "modules", Fingerprint: "a"}, {Module: "idx", Local: "mod/ules", Fingerprint: "a"}, {Module: "idx", Local: "modules", Fingerprint: ""}} {
|
|
if _, err := k.SnapshotKept(context.Background(), r); err == nil {
|
|
t.Errorf("%+v was taken", r)
|
|
}
|
|
}
|
|
}},
|
|
{"a restore sets the live data aside whole, removes it, restores the archive and sets back the other consumers", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
p := take(t, k)
|
|
r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: p.Dir, Caller: "ace.restic", Why: "revert"}
|
|
if _, err := k.RestoreKept(context.Background(), r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !kb.streams["KV_idx_modules"] || len(kb.restored) != 1 {
|
|
t.Errorf("streams %v restored %v", kb.streams, kb.restored)
|
|
}
|
|
if r.CatchUp != "indexer_take_in" {
|
|
t.Errorf("the catch-up tool is %q", r.CatchUp)
|
|
}
|
|
aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
|
|
if err != nil || len(aside) != 1 {
|
|
t.Fatalf("aside: %v %v", aside, err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", aside[0].Name(), ArchiveFile)); err != nil {
|
|
t.Error("the live data's archive is not kept")
|
|
}
|
|
if _, _, found, _ := store.Get(StateKept, "aside."+aside[0].Name()); !found {
|
|
t.Error("what is set aside is not recorded")
|
|
}
|
|
// Only the consumer on a stream the point did not carry is made again; a stream's own comes with its snapshot.
|
|
if len(kb.set) != 1 || kb.set[0].Stream != "forge" || kb.set[0].AckFloor != 41 {
|
|
t.Errorf("positions set back: %+v", kb.set)
|
|
}
|
|
}},
|
|
{"a restore point of another fingerprint or a linked file is refused before anything is removed", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
p := take(t, k)
|
|
bad := *p
|
|
bad.Fingerprint = "other"
|
|
if _, err := k.RestoreKept(context.Background(), &bad); err == nil || !kb.streams["KV_idx_modules"] {
|
|
t.Errorf("err = %v, live %v", err, kb.streams)
|
|
}
|
|
link := filepath.Join(p.Dir, ArchiveFile)
|
|
real := filepath.Join(p.Dir, "real.tar")
|
|
if err := os.Rename(link, real); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(real, link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
|
|
if _, err := k.RestoreKept(context.Background(), r); err == nil || !strings.Contains(err.Error(), "plain file") || !kb.streams["KV_idx_modules"] {
|
|
t.Errorf("err = %v, live %v", err, kb.streams)
|
|
}
|
|
}},
|
|
{"a live data that cannot be set aside stays where it is", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
p := take(t, k)
|
|
k.Bus.(*tarBus).fakeBus.refuse["snapshot"] = errors.New("the server refused")
|
|
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
|
|
if _, err := k.RestoreKept(context.Background(), r); err == nil || !kb.streams["KV_idx_modules"] || !strings.Contains(err.Error(), "nothing was removed") {
|
|
t.Errorf("err = %v, live %v", err, kb.streams)
|
|
}
|
|
}},
|
|
{"a restore that fails puts the live data back and says so", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
p := take(t, k)
|
|
kb.refuse = errors.New("the server refused the archive")
|
|
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
|
|
_, err := k.RestoreKept(context.Background(), r)
|
|
if err == nil || !strings.Contains(err.Error(), "the live data is back as it was") {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
}},
|
|
{"set-aside data goes only on a call that says why and is recorded first", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
|
p := take(t, k)
|
|
if _, err := k.RestoreKept(context.Background(), &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entries, _ := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
|
|
id := entries[0].Name()
|
|
if _, err := k.DiscardAside(&Request{From: id}); err == nil {
|
|
t.Error("discarded with no why")
|
|
}
|
|
if _, err := k.DiscardAside(&Request{From: "../points", Why: "x"}); err == nil {
|
|
t.Error("discarded a path")
|
|
}
|
|
k.Act = func(string, string, string) error { return errors.New("the log is down") }
|
|
r := &Request{From: id, Why: "the revert held", Caller: "ace.operator"}
|
|
k.record(r, "x") // sets Unrecorded as a failing log does
|
|
if _, err := k.DiscardAside(r); err == nil {
|
|
t.Error("discarded with the act unrecorded")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err != nil {
|
|
t.Error("the data went with its act unrecorded")
|
|
}
|
|
k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil }
|
|
if _, err := k.DiscardAside(&Request{From: id, Why: "the revert held", Caller: "ace.operator"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err == nil {
|
|
t.Error("still there")
|
|
}
|
|
if _, _, found, _ := store.Get(StateKept, "aside."+id); found {
|
|
t.Error("its record stays")
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
k, kb, store, acts := keptKeeper(t)
|
|
row.run(t, k, kb, store, acts)
|
|
})
|
|
}
|
|
}
|