Files
mesh-catalog/modules/mesh-recovery/internal/recovery/keeper_test.go
T
jschoubben bf7d040489 mesh-recovery A-143: the dump wait and the snapshot-restored wait are bound keys
Each wait's bound is a key of the module's state bounds with its own time to
live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's
watch, so no timer stays in the process.

Builds: design 47 G A-143, A-77, A-138, §681, §561

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 02:24:44 +01:00

217 lines
8.0 KiB
Go

package recovery_test
import (
"context"
"crypto/sha256"
"encoding/hex"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
"mesh-recovery/internal/recovery"
)
// fakeKeeper is the bus module's keeper holding one snapshot of app_notes (the fake bus's file form). in place: how its
// restore in place ends: "done", "failed", "silent" (it says nothing until cancelled, then failed) or "deaf" (never).
type fakeKeeper struct {
mu sync.Mutex
b *fakeBus
file string
protected bool
busy string
damaged bool
inPlace string
asked []string
wrote int
cancelled chan struct{}
}
func newKeeper(b *fakeBus) *fakeKeeper {
return &fakeKeeper{b: b, file: "snap:app_notes:as dropped", inPlace: "done", cancelled: make(chan struct{})}
}
func (k *fakeKeeper) note(s string) {
k.mu.Lock()
k.asked = append(k.asked, s)
k.mu.Unlock()
}
func (k *fakeKeeper) Get(_ context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) {
k.note("get " + request)
if k.protected {
return meshbroker.SnapshotGetAnswer{Error: "snapshot protected"}, nil
}
sum := sha256.Sum256([]byte(k.file))
ans := meshbroker.SnapshotGetAnswer{Request: request, Module: "app", Local: "notes", Stream: "app_notes", Size: int64(len(k.file)),
Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 1}
if err := fits(ans); err != nil {
return ans, err
}
body := k.file
if k.damaged {
body = "snap:app_notes:altered on its way"
}
n, err := io.WriteString(w, body)
k.wrote += n
return ans, err
}
func (k *fakeKeeper) Restore(_ context.Context, request string, verify bool, _ string) (meshbroker.SnapshotRestoreAnswer, error) {
if verify {
k.note("verify " + request)
return meshbroker.SnapshotRestoreAnswer{Accepted: request}, nil
}
k.note("restore " + request)
if k.busy != "" {
return meshbroker.SnapshotRestoreAnswer{Busy: k.busy}, nil
}
return meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-act-1"}, nil
}
func (k *fakeKeeper) CancelRestore(_ context.Context, act, _ string) (meshbroker.SnapshotRestoreCancelAnswer, error) {
k.note("cancel " + act)
close(k.cancelled)
return meshbroker.SnapshotRestoreCancelAnswer{Accepted: act}, nil
}
func (k *fakeKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) {
ev := meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "the server refused the archive"}
switch k.inPlace {
case "done":
k.b.mu.Lock()
if _, live := k.b.streams["app_notes"]; live {
k.b.mu.Unlock()
return meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "names live"}, true
}
k.b.streams["app_notes"] = "as dropped"
k.b.mu.Unlock()
ev.Outcome, ev.Error = "done", ""
return ev, true
case "failed":
return ev, true
case "silent":
select {
case <-k.cancelled:
ev.Error = "cancelled"
return ev, true
case <-ctx.Done():
return ev, false
}
}
<-ctx.Done()
return ev, false
}
func requestRig(t *testing.T) (*rig, *fakeKeeper) {
r := newRig(t)
k := newKeeper(r.b)
r.e.Keeper = k
r.e.Settings.RestoreWait = 20 * time.Millisecond
return r, k
}
func requestAct() recovery.Act {
return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Caller: "tool-runner", Asker: "op@node"}
}
// A keeper's snapshot by request is read through snapshot-get into incoming/<request>, refused before the first chunk past
// the cap, checked against its digest before any aside, restored by the module itself, and its file deleted at the end
// (design 47 G §681 Its use, §682).
func TestARestoreByRequestReadsTheKeepersSnapshot(t *testing.T) {
for _, tc := range []struct {
name string
cap int64
damaged bool
err string
}{
{name: "received and restored", cap: 1 << 20},
{name: "altered on its way", cap: 1 << 20, damaged: true, err: "snapshot damaged"},
{name: "past the cap", cap: 10, err: "cap:"},
} {
t.Run(tc.name, func(t *testing.T) {
r, k := requestRig(t)
r.e.Settings.Cap, k.damaged = tc.cap, tc.damaged
ans := r.do(requestAct())
ev := lastRestored(t, r)
if ev.Act != ans.Accepted || !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") {
t.Fatalf("restored %+v, want error %q", ev, tc.err)
}
if left, _ := os.ReadDir(filepath.Join(r.e.Dir, "incoming")); len(left) != 0 {
t.Errorf("incoming holds %v after the act", left)
}
kp, _ := r.e.KeptPoints("app", "notes")
if tc.err != "" {
if r.b.streams["app_notes"] != "hello" || len(kp.Asides) != 0 {
t.Errorf("a refused restore touched the live data: %v, asides %+v", r.b.streams, kp.Asides)
}
if tc.name == "past the cap" && k.wrote != 0 {
t.Errorf("%d bytes received past the cap", k.wrote)
}
return
}
if r.b.streams["app_notes"] != "as dropped" || r.b.streams["KV_app_seen"] != "world" || len(kp.Asides) != 1 ||
ev.Aside == nil || len(ev.Aside.Names) != 1 || ev.Aside.Names[0] != "app_notes" {
t.Errorf("streams %v, asides %+v, event aside %+v", r.b.streams, kp.Asides, ev.Aside)
}
})
}
}
// A protected snapshot is verified, the running data of the module and kept data set aside (A-137), and restored in place
// by the keeper; its failure puts the aside back; at the bound the module cancels first and puts the aside back on the
// cancel's outcome, and with no outcome at all ends `snapshot-restore unanswered`, the aside left unrestored; a busy keeper
// ends the act before any aside (§681, A-138, A-140).
func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) {
for _, tc := range []struct {
inPlace, busy string
err string
asked string
live string
unrestored int
bounds string // how each bound key restore-wait.<act> was removed (A-143)
}{
{inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped", bounds: "ended"},
{inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed", bounds: "ended"},
{inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled", bounds: "expired|ended"},
{inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1, bounds: "expired|expired"},
{inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0", bounds: "ended"},
} {
t.Run(tc.inPlace+tc.busy, func(t *testing.T) {
r, k := requestRig(t)
k.protected, k.inPlace, k.busy = true, tc.inPlace, tc.busy
r.do(requestAct())
ev := lastRestored(t, r)
if !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") || ev.Fingerprint != "fp-run" {
t.Fatalf("restored %+v, want error %q", ev, tc.err)
}
if got := strings.Join(k.asked, "|"); got != tc.asked {
t.Errorf("asked %s, want %s", got, tc.asked)
}
if r.b.streams["app_notes"] != tc.live {
t.Errorf("app_notes is %q, want %q", r.b.streams["app_notes"], tc.live)
}
if tc.live == "hello" && r.b.streams["KV_app_seen"] != "world" {
t.Errorf("the second name of the running data is not put back: %v", r.b.streams)
}
if n := len(r.e.Unrestored()); n != tc.unrestored {
t.Errorf("%d unrestored asides, want %d", n, tc.unrestored)
}
w, rm := r.bounds.keys()
var wantW, wantR []string
for _, by := range strings.Split(tc.bounds, "|") {
wantW = append(wantW, "restore-wait."+ev.Act+" 20ms")
wantR = append(wantR, "restore-wait."+ev.Act+" "+by)
}
if w != strings.Join(wantW, "|") || rm != strings.Join(wantR, "|") {
t.Errorf("bound keys written %q, removed %q; want %q, %q", w, rm, strings.Join(wantW, "|"), strings.Join(wantR, "|"))
}
})
}
}