Files
mesh-catalog/modules/mesh-recovery/internal/recovery/restore_test.go
T
jschoubben bf7d040489 mesh-recovery A-143: the dump wait and the snapshot-restored wait are bound keys
Each wait's bound is a key of the module's state bounds with its own time to
live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's
watch, so no timer stays in the process.

Builds: design 47 G A-143, A-77, A-138, §681, §561

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 02:24:44 +01:00

350 lines
14 KiB
Go

package recovery_test
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"mesh-recovery/internal/recovery"
)
func restoreAct(snapshot string) recovery.Act {
return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Snapshot: snapshot, Why: "revert", Caller: "node.engine"}
}
func takenRig(t *testing.T) *rig {
r := newRig(t)
r.do(takeAct())
r.s.events = nil
return r
}
func lastRestored(t *testing.T, r *rig) seat.RestoredEvent {
t.Helper()
ev := r.s.named(seat.EventRestored)
if len(ev) != 1 {
t.Fatalf("restored events %+v", r.s.events)
}
return ev[0].Body.(seat.RestoredEvent)
}
// A restore says restore-started, sets the live data aside whole, removes it, restores the point under its names and sets
// each consumer to its recorded sequence, and says restored with the aside (design 47 G §516, §560).
func TestARestoreSetsTheLiveDataAsideAndRestoresThePoint(t *testing.T) {
r := takenRig(t)
r.b.streams["app_notes"] = "changed since"
ans := r.do(restoreAct("app/notes/2-fp1"))
ev := lastRestored(t, r)
if ev.Outcome != "done" || ev.Act != ans.Accepted || ev.Aside == nil || ev.Aside.Size == 0 {
t.Fatalf("restored %+v", ev)
}
if r.b.streams["app_notes"] != "hello" || r.b.streams["KV_app_seen"] != "world" {
t.Errorf("streams %v", r.b.streams)
}
if len(r.b.setBack) != 1 || r.b.setBack[0].Sequence != 7 {
t.Errorf("consumers %+v", r.b.setBack)
}
if len(r.s.named(seat.EventRestoreStarted)) != 1 {
t.Errorf("no restore-started")
}
kp, _ := r.e.KeptPoints("app", "notes")
if len(kp.Asides) != 1 || len(kp.Points) != 1 || kp.Running != nil {
t.Errorf("kept points %+v", kp)
}
}
// A point missing from the directory says missing; with from-backup the backup holder brings it back and it is restored
// once its digest is checked; when no repository snapshot holds it, restore point missing (§516, §567).
func TestAMissingPointIsBroughtBackOrSaidMissing(t *testing.T) {
r := takenRig(t)
back := filepath.Join(t.TempDir(), "back")
if err := os.Rename(filepath.Join(r.e.Dir, "points"), back); err != nil {
t.Fatal(err)
}
r.do(restoreAct("app/notes/2-fp1"))
if ev := lastRestored(t, r); !ev.Missing || ev.Outcome != "failed" || ev.Error != recovery.ErrMissing {
t.Errorf("no backup asked: %+v", ev)
}
r.s.events = nil
a := restoreAct("app/notes/2-fp1")
a.FromBackup = true
r.do(a)
if ev := lastRestored(t, r); !ev.Missing || ev.Error != recovery.ErrMissing {
t.Errorf("backup holder holds none: %+v", ev)
}
r.s.events, r.s.backup = nil, back
r.do(a)
if ev := lastRestored(t, r); ev.Outcome != "done" {
t.Errorf("brought back: %+v", ev)
}
}
// A point whose bytes differ from the digests on the bus is refused before any aside is taken, the record in the
// directory never being the reference (§567).
func TestAPointWhoseBytesWereAlteredIsRefusedBeforeAnyAside(t *testing.T) {
r := takenRig(t)
snap := filepath.Join(r.e.Dir, "points", "app", "notes", "2-fp1", "app_notes.snap")
if err := os.WriteFile(snap, []byte("snap:app_notes:evil"), 0o600); err != nil {
t.Fatal(err)
}
r.do(restoreAct("app/notes/2-fp1"))
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrDamaged) {
t.Errorf("restored %+v", ev)
}
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 || r.b.streams["app_notes"] != "hello" {
t.Errorf("an aside was taken or the live data changed: %+v %v", kp, r.b.streams)
}
}
// A step after the removal that fails puts the aside back under the live names and the act ends failed, so the kept data
// is never left gone (§516).
func TestARestoreThatFailsAfterTheRemovalPutsTheAsideBack(t *testing.T) {
r := takenRig(t)
r.b.streams["app_notes"] = "live now"
r.b.failRestores = 1
r.do(restoreAct("app/notes/2-fp1"))
if ev := lastRestored(t, r); ev.Outcome != "failed" || ev.Aside == nil {
t.Fatalf("restored %+v", ev)
}
if r.b.streams["app_notes"] != "live now" || r.b.streams["KV_app_seen"] != "world" {
t.Errorf("live data not as before: %v", r.b.streams)
}
if n := len(r.e.Unrestored()); n != 0 {
t.Errorf("%d asides unrestored", n)
}
}
// A restore past the cap is refused with the live data untouched (§564).
func TestARestoreWhoseAsideWouldPassTheCapIsRefusedBeforeTheRemoval(t *testing.T) {
r := takenRig(t)
r.e.Settings.Cap = 1
r.do(restoreAct("app/notes/2-fp1"))
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) {
t.Errorf("restored %+v", ev)
}
if r.b.streams["app_notes"] != "hello" {
t.Errorf("live data touched: %v", r.b.streams)
}
}
// An act waiting on the dump is cancelled with its ending event failed/cancelled and the hand act's words as why; an act
// that has begun is refused, naming its step (§566, A-117).
func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) {
r := takenRig(t)
hd := holdDump{waiting: make(chan string, 1)}
r.e.Dump = hd
// Past the bound (a setting, §557) the waiting act writes recovery-waits, cleared once it ends (§561).
fs := newHeldFindings()
r.e.Findings, r.e.Settings.DumpWait = fs, time.Millisecond
ans := r.e.Accept(restoreAct("app/notes/2-fp1"))
<-hd.waiting
if f := <-fs.written; f.Kind != recovery.FindingRecoveryWaits || f.Scope != "app" || f.ID != ans.Accepted {
t.Errorf("written %+v", f)
}
kp, _ := r.e.KeptPoints("", "")
if kp.Running == nil || kp.Running.WaitsOn == "" || kp.Running.Act != ans.Accepted {
t.Errorf("running %+v", kp.Running)
}
if c := r.e.Cancel("act-nothing", "x", "tools", "op"); c.Error == "" {
t.Errorf("an unknown act cancelled: %+v", c)
}
if c := r.e.Cancel(ans.Accepted, "not now", "tools", "op"); c.Accepted != ans.Accepted {
t.Errorf("cancel %+v", c)
}
r.e.Wait()
ev := lastRestored(t, r)
if ev.Outcome != "failed" || ev.Error != recovery.ErrCanceled || ev.Why != "not now" || ev.Act != ans.Accepted {
t.Errorf("restored %+v", ev)
}
if fs.has(recovery.FindingRecoveryWaits) {
t.Errorf("recovery-waits stands after the act ended: %+v", fs.Held())
}
// The wait's bound is a key of the state bounds, its expiry what wrote recovery-waits (A-143).
if w, rm := r.bounds.keys(); w != "dump-wait."+ans.Accepted+" 1ms" || rm != "dump-wait."+ans.Accepted+" expired" {
t.Errorf("bound keys written %q, removed %q", w, rm)
}
}
// retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its
// last act did not fail (§682).
func TestRetryAsksTheLastFailedActAgain(t *testing.T) {
r := takenRig(t)
r.e.Settings.Cap = 1
r.do(restoreAct("app/notes/2-fp1"))
r.e.Settings.Cap = 1 << 20
r.s.events = nil
ans := r.e.Retry("app", "notes", "again", "tools", "op")
r.e.Wait()
if ev := lastRestored(t, r); ans.Accepted == "" || ev.Outcome != "done" || ev.Act != ans.Accepted {
t.Errorf("answer %+v restored %+v", ans, ev)
}
if again := r.e.Retry("app", "notes", "again", "tools", "op"); again.Error != "nothing to retry" {
t.Errorf("answer %+v", again)
}
}
// An aside whose record ends at removed is put back at the start, an act cut off by a restart ends failed with stopped,
// and a point left under a temporary name is removed (§516, §560, §564).
func TestAStartPutsBackAnAsideEndsACutOffActAndClearsTemporaryPoints(t *testing.T) {
r := takenRig(t)
r.do(restoreAct("app/notes/2-fp1")) // leaves an aside at "restored"
_, recs, _ := r.e.Asides()
if len(recs) != 1 {
t.Fatal("no aside")
}
// Cut a restore off after its removal: the record says removed, the live data is gone, an accepted act never ended.
dir := filepath.Join(r.e.Dir, "aside", "app", "notes", recs[0].ID)
raw, _ := os.ReadFile(filepath.Join(dir, "aside.json"))
_ = os.WriteFile(filepath.Join(dir, "aside.json"), []byte(strings.Replace(string(raw), `"step": "restored"`, `"step": "removed"`, 1)), 0o600)
delete(r.b.streams, "app_notes")
delete(r.b.streams, "KV_app_seen")
tmp := filepath.Join(r.e.Dir, "points", "app", "notes", ".tmp-act-x")
_ = os.MkdirAll(tmp, 0o700)
_ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-cut.json"), []byte(`{"act":"act-cut","kind":"restore","module":"app","kept":"notes","why":"w","caller":"c","since":"2026-11-12T00:00:00Z"}`), 0o600)
r.s.events = nil
if left := r.e.Start(context.Background()); len(left) != 0 {
t.Errorf("unrestored %+v", left)
}
if r.b.streams["app_notes"] != "hello" {
t.Errorf("the aside was not put back: %v", r.b.streams)
}
if _, err := os.Stat(tmp); err == nil {
t.Errorf("the temporary point stays")
}
if ev := lastRestored(t, r); ev.Act != "act-cut" || ev.Error != recovery.ErrStopped {
t.Errorf("restored %+v", ev)
}
}
// restore-aside puts an aside back under the live names, refused while live objects of those names exist; discard-aside
// removes the aside's directory and says aside-discarded (§554, §566).
func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testing.T) {
r := takenRig(t)
r.do(restoreAct("app/notes/2-fp1"))
_, recs, _ := r.e.Asides()
id := recs[0].ID
r.s.events = nil
r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"})
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "live stream") {
t.Errorf("restored %+v", ev)
}
r.s.events = nil
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: "../x", Why: "x", Caller: "tools"})
if len(r.s.named(seat.EventAsideDiscarded)) != 0 {
t.Errorf("a path outside was discarded")
}
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"})
if ev := r.s.named(seat.EventAsideDiscarded); len(ev) != 1 || ev[0].Body.(seat.AsideDiscardedEvent).ID != id {
t.Errorf("events %+v", r.s.events)
}
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 {
t.Errorf("asides %+v", kp.Asides)
}
}
// The hand verb aside sets the live data aside whole and removes it, naming the streams as a list, and restore-aside puts
// it back once no live name exists (§516, §554, §566, A-123, A-124).
func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) {
r := newRig(t)
// The findings of §561: aside-unrestored while the aside alone holds the data, points-filling past the fill mark.
fs := newHeldFindings()
r.e.Findings, r.e.Node, r.e.Settings.FillingMark, r.e.Settings.Cap = fs, "srv", 1, 4096
ans := r.do(recovery.Act{Kind: recovery.KindAside, Module: "app", Kept: "notes", Why: "look at it", Caller: "tools", Asker: "op"})
ev := r.s.named(seat.EventSetAside)
if ans.Accepted == "" || len(ev) != 1 {
t.Fatalf("answer %+v events %+v", ans, r.s.events)
}
sa := ev[0].Body.(seat.SetAsideEvent)
if sa.Act != ans.Accepted || len(sa.Names) != 2 || sa.Size == 0 || len(r.b.streams) != 0 {
t.Errorf("set-aside %+v streams %v", sa, r.b.streams)
}
if got := r.e.Unrestored(); len(got) != 1 {
t.Errorf("unrestored %+v", got)
}
// kept-points says it, which the tool and the health check read (A-129).
if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID {
t.Errorf("kept-points unrestored %+v", kp.Unrestored)
}
if !fs.has(recovery.FindingAsideUnrestored) || !fs.has(recovery.FindingPointsFilling) {
t.Errorf("findings %+v", fs.Held())
}
r.s.events = nil
r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"})
if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 {
t.Errorf("streams %v", r.b.streams)
}
r.e.Settings.FillingMark = 100
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "not needed", Caller: "tools"})
if fs.has(recovery.FindingAsideUnrestored) || fs.has(recovery.FindingPointsFilling) {
t.Errorf("findings stand after the aside was restored and discarded: %+v", fs.Held())
}
}
// Said means acknowledged: a restore whose restore-started the bus did not take reads nothing and sets nothing aside, and
// ends failed (§563).
func TestARestoreWhoseStartIsNotAcknowledgedTouchesNothing(t *testing.T) {
r := takenRig(t)
r.s.sayFail = map[string]error{seat.EventRestoreStarted: errTest}
r.do(restoreAct("app/notes/2-fp1"))
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "restore-started") || r.b.streams["app_notes"] != "hello" {
t.Errorf("restored %+v, streams %v", ev, r.b.streams)
}
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 {
t.Errorf("an aside was taken: %+v", kp.Asides)
}
}
// stallBus blocks its first Restore until the act's context ends, as a restore under way when the process stops.
type stallBus struct {
*fakeBus
in chan struct{}
stalled bool
}
func (s *stallBus) Restore(ctx context.Context, file string) error {
if !s.stalled {
s.stalled = true
close(s.in)
<-ctx.Done()
return ctx.Err()
}
return s.fakeBus.Restore(ctx, file)
}
// A stop of the process (a move's or a restart's) answers every new call an error naming it, ends the running restore
// through its failure path, putting the aside back, and ends every accepted act with `stopped`, so its caller asks again
// (design 47 G §562, A-141).
func TestAStopEndsTheRunningActThroughItsFailurePathAndStopped(t *testing.T) {
r := takenRig(t)
r.b.streams["app_notes"] = "changed since"
stall := &stallBus{fakeBus: r.b, in: make(chan struct{})}
r.e.Bus = stall
ans := r.e.Accept(restoreAct("app/notes/2-fp1"))
<-stall.in
// An accepted act a crash left unended is ended by the stop too.
_ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-0.json"), []byte(`{"act":"act-0","kind":"take","module":"app","kept":"seen","why":"w","caller":"c","since":"2026-11-11T00:00:00Z"}`), 0o600)
r.e.Stop("a move")
if r.b.streams["app_notes"] != "changed since" || r.b.streams["KV_app_seen"] != "world" {
t.Errorf("the aside is not put back: %v", r.b.streams)
}
if ev := lastRestored(t, r); ev.Act != ans.Accepted || ev.Outcome != "failed" || ev.Error != recovery.ErrStopped {
t.Errorf("restored %+v", ev)
}
ended := r.s.named(seat.EventEnded)
if len(ended) != 1 || ended[0].Body.(seat.EndedEvent).Act != "act-0" || ended[0].Body.(seat.EndedEvent).Error != recovery.ErrStopped {
t.Errorf("ended %+v", ended)
}
if got := r.e.Accept(takeAct()); !strings.Contains(got.Error, "stopping (a move)") {
t.Errorf("a call during the stop answered %+v", got)
}
verb := r.e.Handlers()[seat.VerbKeptPoints]
if got, _ := verb("c", []byte(`{}`)); !strings.Contains(got.(recovery.Answer).Error, "stopping") {
t.Errorf("kept-points during the stop answered %+v", got)
}
}