Each wait's bound is a key of the module's state bounds with its own time to live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's watch, so no timer stays in the process. Builds: design 47 G A-143, A-77, A-138, §681, §561 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
217 lines
8.0 KiB
Go
217 lines
8.0 KiB
Go
package recovery_test
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
|
|
|
"mesh-recovery/internal/recovery"
|
|
)
|
|
|
|
// fakeKeeper is the bus module's keeper holding one snapshot of app_notes (the fake bus's file form). in place: how its
|
|
// restore in place ends: "done", "failed", "silent" (it says nothing until cancelled, then failed) or "deaf" (never).
|
|
type fakeKeeper struct {
|
|
mu sync.Mutex
|
|
b *fakeBus
|
|
file string
|
|
protected bool
|
|
busy string
|
|
damaged bool
|
|
inPlace string
|
|
asked []string
|
|
wrote int
|
|
cancelled chan struct{}
|
|
}
|
|
|
|
func newKeeper(b *fakeBus) *fakeKeeper {
|
|
return &fakeKeeper{b: b, file: "snap:app_notes:as dropped", inPlace: "done", cancelled: make(chan struct{})}
|
|
}
|
|
|
|
func (k *fakeKeeper) note(s string) {
|
|
k.mu.Lock()
|
|
k.asked = append(k.asked, s)
|
|
k.mu.Unlock()
|
|
}
|
|
|
|
func (k *fakeKeeper) Get(_ context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) {
|
|
k.note("get " + request)
|
|
if k.protected {
|
|
return meshbroker.SnapshotGetAnswer{Error: "snapshot protected"}, nil
|
|
}
|
|
sum := sha256.Sum256([]byte(k.file))
|
|
ans := meshbroker.SnapshotGetAnswer{Request: request, Module: "app", Local: "notes", Stream: "app_notes", Size: int64(len(k.file)),
|
|
Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 1}
|
|
if err := fits(ans); err != nil {
|
|
return ans, err
|
|
}
|
|
body := k.file
|
|
if k.damaged {
|
|
body = "snap:app_notes:altered on its way"
|
|
}
|
|
n, err := io.WriteString(w, body)
|
|
k.wrote += n
|
|
return ans, err
|
|
}
|
|
|
|
func (k *fakeKeeper) Restore(_ context.Context, request string, verify bool, _ string) (meshbroker.SnapshotRestoreAnswer, error) {
|
|
if verify {
|
|
k.note("verify " + request)
|
|
return meshbroker.SnapshotRestoreAnswer{Accepted: request}, nil
|
|
}
|
|
k.note("restore " + request)
|
|
if k.busy != "" {
|
|
return meshbroker.SnapshotRestoreAnswer{Busy: k.busy}, nil
|
|
}
|
|
return meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-act-1"}, nil
|
|
}
|
|
|
|
func (k *fakeKeeper) CancelRestore(_ context.Context, act, _ string) (meshbroker.SnapshotRestoreCancelAnswer, error) {
|
|
k.note("cancel " + act)
|
|
close(k.cancelled)
|
|
return meshbroker.SnapshotRestoreCancelAnswer{Accepted: act}, nil
|
|
}
|
|
|
|
func (k *fakeKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) {
|
|
ev := meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "the server refused the archive"}
|
|
switch k.inPlace {
|
|
case "done":
|
|
k.b.mu.Lock()
|
|
if _, live := k.b.streams["app_notes"]; live {
|
|
k.b.mu.Unlock()
|
|
return meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "names live"}, true
|
|
}
|
|
k.b.streams["app_notes"] = "as dropped"
|
|
k.b.mu.Unlock()
|
|
ev.Outcome, ev.Error = "done", ""
|
|
return ev, true
|
|
case "failed":
|
|
return ev, true
|
|
case "silent":
|
|
select {
|
|
case <-k.cancelled:
|
|
ev.Error = "cancelled"
|
|
return ev, true
|
|
case <-ctx.Done():
|
|
return ev, false
|
|
}
|
|
}
|
|
<-ctx.Done()
|
|
return ev, false
|
|
}
|
|
|
|
func requestRig(t *testing.T) (*rig, *fakeKeeper) {
|
|
r := newRig(t)
|
|
k := newKeeper(r.b)
|
|
r.e.Keeper = k
|
|
r.e.Settings.RestoreWait = 20 * time.Millisecond
|
|
return r, k
|
|
}
|
|
|
|
func requestAct() recovery.Act {
|
|
return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Caller: "tool-runner", Asker: "op@node"}
|
|
}
|
|
|
|
// A keeper's snapshot by request is read through snapshot-get into incoming/<request>, refused before the first chunk past
|
|
// the cap, checked against its digest before any aside, restored by the module itself, and its file deleted at the end
|
|
// (design 47 G §681 Its use, §682).
|
|
func TestARestoreByRequestReadsTheKeepersSnapshot(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
cap int64
|
|
damaged bool
|
|
err string
|
|
}{
|
|
{name: "received and restored", cap: 1 << 20},
|
|
{name: "altered on its way", cap: 1 << 20, damaged: true, err: "snapshot damaged"},
|
|
{name: "past the cap", cap: 10, err: "cap:"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
r, k := requestRig(t)
|
|
r.e.Settings.Cap, k.damaged = tc.cap, tc.damaged
|
|
ans := r.do(requestAct())
|
|
ev := lastRestored(t, r)
|
|
if ev.Act != ans.Accepted || !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") {
|
|
t.Fatalf("restored %+v, want error %q", ev, tc.err)
|
|
}
|
|
if left, _ := os.ReadDir(filepath.Join(r.e.Dir, "incoming")); len(left) != 0 {
|
|
t.Errorf("incoming holds %v after the act", left)
|
|
}
|
|
kp, _ := r.e.KeptPoints("app", "notes")
|
|
if tc.err != "" {
|
|
if r.b.streams["app_notes"] != "hello" || len(kp.Asides) != 0 {
|
|
t.Errorf("a refused restore touched the live data: %v, asides %+v", r.b.streams, kp.Asides)
|
|
}
|
|
if tc.name == "past the cap" && k.wrote != 0 {
|
|
t.Errorf("%d bytes received past the cap", k.wrote)
|
|
}
|
|
return
|
|
}
|
|
if r.b.streams["app_notes"] != "as dropped" || r.b.streams["KV_app_seen"] != "world" || len(kp.Asides) != 1 ||
|
|
ev.Aside == nil || len(ev.Aside.Names) != 1 || ev.Aside.Names[0] != "app_notes" {
|
|
t.Errorf("streams %v, asides %+v, event aside %+v", r.b.streams, kp.Asides, ev.Aside)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A protected snapshot is verified, the running data of the module and kept data set aside (A-137), and restored in place
|
|
// by the keeper; its failure puts the aside back; at the bound the module cancels first and puts the aside back on the
|
|
// cancel's outcome, and with no outcome at all ends `snapshot-restore unanswered`, the aside left unrestored; a busy keeper
|
|
// ends the act before any aside (§681, A-138, A-140).
|
|
func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
inPlace, busy string
|
|
err string
|
|
asked string
|
|
live string
|
|
unrestored int
|
|
bounds string // how each bound key restore-wait.<act> was removed (A-143)
|
|
}{
|
|
{inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped", bounds: "ended"},
|
|
{inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed", bounds: "ended"},
|
|
{inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled", bounds: "expired|ended"},
|
|
{inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1, bounds: "expired|expired"},
|
|
{inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0", bounds: "ended"},
|
|
} {
|
|
t.Run(tc.inPlace+tc.busy, func(t *testing.T) {
|
|
r, k := requestRig(t)
|
|
k.protected, k.inPlace, k.busy = true, tc.inPlace, tc.busy
|
|
r.do(requestAct())
|
|
ev := lastRestored(t, r)
|
|
if !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") || ev.Fingerprint != "fp-run" {
|
|
t.Fatalf("restored %+v, want error %q", ev, tc.err)
|
|
}
|
|
if got := strings.Join(k.asked, "|"); got != tc.asked {
|
|
t.Errorf("asked %s, want %s", got, tc.asked)
|
|
}
|
|
if r.b.streams["app_notes"] != tc.live {
|
|
t.Errorf("app_notes is %q, want %q", r.b.streams["app_notes"], tc.live)
|
|
}
|
|
if tc.live == "hello" && r.b.streams["KV_app_seen"] != "world" {
|
|
t.Errorf("the second name of the running data is not put back: %v", r.b.streams)
|
|
}
|
|
if n := len(r.e.Unrestored()); n != tc.unrestored {
|
|
t.Errorf("%d unrestored asides, want %d", n, tc.unrestored)
|
|
}
|
|
w, rm := r.bounds.keys()
|
|
var wantW, wantR []string
|
|
for _, by := range strings.Split(tc.bounds, "|") {
|
|
wantW = append(wantW, "restore-wait."+ev.Act+" 20ms")
|
|
wantR = append(wantR, "restore-wait."+ev.Act+" "+by)
|
|
}
|
|
if w != strings.Join(wantW, "|") || rm != strings.Join(wantR, "|") {
|
|
t.Errorf("bound keys written %q, removed %q; want %q, %q", w, rm, strings.Join(wantW, "|"), strings.Join(wantR, "|"))
|
|
}
|
|
})
|
|
}
|
|
}
|