Each wait's bound is a key of the module's state bounds with its own time to live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's watch, so no timer stays in the process. Builds: design 47 G A-143, A-77, A-138, §681, §561 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
350 lines
14 KiB
Go
350 lines
14 KiB
Go
package recovery_test
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
|
|
|
"mesh-recovery/internal/recovery"
|
|
)
|
|
|
|
func restoreAct(snapshot string) recovery.Act {
|
|
return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Snapshot: snapshot, Why: "revert", Caller: "node.engine"}
|
|
}
|
|
|
|
func takenRig(t *testing.T) *rig {
|
|
r := newRig(t)
|
|
r.do(takeAct())
|
|
r.s.events = nil
|
|
return r
|
|
}
|
|
|
|
func lastRestored(t *testing.T, r *rig) seat.RestoredEvent {
|
|
t.Helper()
|
|
ev := r.s.named(seat.EventRestored)
|
|
if len(ev) != 1 {
|
|
t.Fatalf("restored events %+v", r.s.events)
|
|
}
|
|
return ev[0].Body.(seat.RestoredEvent)
|
|
}
|
|
|
|
// A restore says restore-started, sets the live data aside whole, removes it, restores the point under its names and sets
|
|
// each consumer to its recorded sequence, and says restored with the aside (design 47 G §516, §560).
|
|
func TestARestoreSetsTheLiveDataAsideAndRestoresThePoint(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.b.streams["app_notes"] = "changed since"
|
|
ans := r.do(restoreAct("app/notes/2-fp1"))
|
|
ev := lastRestored(t, r)
|
|
if ev.Outcome != "done" || ev.Act != ans.Accepted || ev.Aside == nil || ev.Aside.Size == 0 {
|
|
t.Fatalf("restored %+v", ev)
|
|
}
|
|
if r.b.streams["app_notes"] != "hello" || r.b.streams["KV_app_seen"] != "world" {
|
|
t.Errorf("streams %v", r.b.streams)
|
|
}
|
|
if len(r.b.setBack) != 1 || r.b.setBack[0].Sequence != 7 {
|
|
t.Errorf("consumers %+v", r.b.setBack)
|
|
}
|
|
if len(r.s.named(seat.EventRestoreStarted)) != 1 {
|
|
t.Errorf("no restore-started")
|
|
}
|
|
kp, _ := r.e.KeptPoints("app", "notes")
|
|
if len(kp.Asides) != 1 || len(kp.Points) != 1 || kp.Running != nil {
|
|
t.Errorf("kept points %+v", kp)
|
|
}
|
|
}
|
|
|
|
// A point missing from the directory says missing; with from-backup the backup holder brings it back and it is restored
|
|
// once its digest is checked; when no repository snapshot holds it, restore point missing (§516, §567).
|
|
func TestAMissingPointIsBroughtBackOrSaidMissing(t *testing.T) {
|
|
r := takenRig(t)
|
|
back := filepath.Join(t.TempDir(), "back")
|
|
if err := os.Rename(filepath.Join(r.e.Dir, "points"), back); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
if ev := lastRestored(t, r); !ev.Missing || ev.Outcome != "failed" || ev.Error != recovery.ErrMissing {
|
|
t.Errorf("no backup asked: %+v", ev)
|
|
}
|
|
r.s.events = nil
|
|
a := restoreAct("app/notes/2-fp1")
|
|
a.FromBackup = true
|
|
r.do(a)
|
|
if ev := lastRestored(t, r); !ev.Missing || ev.Error != recovery.ErrMissing {
|
|
t.Errorf("backup holder holds none: %+v", ev)
|
|
}
|
|
r.s.events, r.s.backup = nil, back
|
|
r.do(a)
|
|
if ev := lastRestored(t, r); ev.Outcome != "done" {
|
|
t.Errorf("brought back: %+v", ev)
|
|
}
|
|
}
|
|
|
|
// A point whose bytes differ from the digests on the bus is refused before any aside is taken, the record in the
|
|
// directory never being the reference (§567).
|
|
func TestAPointWhoseBytesWereAlteredIsRefusedBeforeAnyAside(t *testing.T) {
|
|
r := takenRig(t)
|
|
snap := filepath.Join(r.e.Dir, "points", "app", "notes", "2-fp1", "app_notes.snap")
|
|
if err := os.WriteFile(snap, []byte("snap:app_notes:evil"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrDamaged) {
|
|
t.Errorf("restored %+v", ev)
|
|
}
|
|
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 || r.b.streams["app_notes"] != "hello" {
|
|
t.Errorf("an aside was taken or the live data changed: %+v %v", kp, r.b.streams)
|
|
}
|
|
}
|
|
|
|
// A step after the removal that fails puts the aside back under the live names and the act ends failed, so the kept data
|
|
// is never left gone (§516).
|
|
func TestARestoreThatFailsAfterTheRemovalPutsTheAsideBack(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.b.streams["app_notes"] = "live now"
|
|
r.b.failRestores = 1
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
if ev := lastRestored(t, r); ev.Outcome != "failed" || ev.Aside == nil {
|
|
t.Fatalf("restored %+v", ev)
|
|
}
|
|
if r.b.streams["app_notes"] != "live now" || r.b.streams["KV_app_seen"] != "world" {
|
|
t.Errorf("live data not as before: %v", r.b.streams)
|
|
}
|
|
if n := len(r.e.Unrestored()); n != 0 {
|
|
t.Errorf("%d asides unrestored", n)
|
|
}
|
|
}
|
|
|
|
// A restore past the cap is refused with the live data untouched (§564).
|
|
func TestARestoreWhoseAsideWouldPassTheCapIsRefusedBeforeTheRemoval(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.e.Settings.Cap = 1
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) {
|
|
t.Errorf("restored %+v", ev)
|
|
}
|
|
if r.b.streams["app_notes"] != "hello" {
|
|
t.Errorf("live data touched: %v", r.b.streams)
|
|
}
|
|
}
|
|
|
|
// An act waiting on the dump is cancelled with its ending event failed/cancelled and the hand act's words as why; an act
|
|
// that has begun is refused, naming its step (§566, A-117).
|
|
func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) {
|
|
r := takenRig(t)
|
|
hd := holdDump{waiting: make(chan string, 1)}
|
|
r.e.Dump = hd
|
|
// Past the bound (a setting, §557) the waiting act writes recovery-waits, cleared once it ends (§561).
|
|
fs := newHeldFindings()
|
|
r.e.Findings, r.e.Settings.DumpWait = fs, time.Millisecond
|
|
ans := r.e.Accept(restoreAct("app/notes/2-fp1"))
|
|
<-hd.waiting
|
|
if f := <-fs.written; f.Kind != recovery.FindingRecoveryWaits || f.Scope != "app" || f.ID != ans.Accepted {
|
|
t.Errorf("written %+v", f)
|
|
}
|
|
kp, _ := r.e.KeptPoints("", "")
|
|
if kp.Running == nil || kp.Running.WaitsOn == "" || kp.Running.Act != ans.Accepted {
|
|
t.Errorf("running %+v", kp.Running)
|
|
}
|
|
if c := r.e.Cancel("act-nothing", "x", "tools", "op"); c.Error == "" {
|
|
t.Errorf("an unknown act cancelled: %+v", c)
|
|
}
|
|
if c := r.e.Cancel(ans.Accepted, "not now", "tools", "op"); c.Accepted != ans.Accepted {
|
|
t.Errorf("cancel %+v", c)
|
|
}
|
|
r.e.Wait()
|
|
ev := lastRestored(t, r)
|
|
if ev.Outcome != "failed" || ev.Error != recovery.ErrCanceled || ev.Why != "not now" || ev.Act != ans.Accepted {
|
|
t.Errorf("restored %+v", ev)
|
|
}
|
|
if fs.has(recovery.FindingRecoveryWaits) {
|
|
t.Errorf("recovery-waits stands after the act ended: %+v", fs.Held())
|
|
}
|
|
// The wait's bound is a key of the state bounds, its expiry what wrote recovery-waits (A-143).
|
|
if w, rm := r.bounds.keys(); w != "dump-wait."+ans.Accepted+" 1ms" || rm != "dump-wait."+ans.Accepted+" expired" {
|
|
t.Errorf("bound keys written %q, removed %q", w, rm)
|
|
}
|
|
}
|
|
|
|
// retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its
|
|
// last act did not fail (§682).
|
|
func TestRetryAsksTheLastFailedActAgain(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.e.Settings.Cap = 1
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
r.e.Settings.Cap = 1 << 20
|
|
r.s.events = nil
|
|
ans := r.e.Retry("app", "notes", "again", "tools", "op")
|
|
r.e.Wait()
|
|
if ev := lastRestored(t, r); ans.Accepted == "" || ev.Outcome != "done" || ev.Act != ans.Accepted {
|
|
t.Errorf("answer %+v restored %+v", ans, ev)
|
|
}
|
|
if again := r.e.Retry("app", "notes", "again", "tools", "op"); again.Error != "nothing to retry" {
|
|
t.Errorf("answer %+v", again)
|
|
}
|
|
}
|
|
|
|
// An aside whose record ends at removed is put back at the start, an act cut off by a restart ends failed with stopped,
|
|
// and a point left under a temporary name is removed (§516, §560, §564).
|
|
func TestAStartPutsBackAnAsideEndsACutOffActAndClearsTemporaryPoints(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.do(restoreAct("app/notes/2-fp1")) // leaves an aside at "restored"
|
|
_, recs, _ := r.e.Asides()
|
|
if len(recs) != 1 {
|
|
t.Fatal("no aside")
|
|
}
|
|
// Cut a restore off after its removal: the record says removed, the live data is gone, an accepted act never ended.
|
|
dir := filepath.Join(r.e.Dir, "aside", "app", "notes", recs[0].ID)
|
|
raw, _ := os.ReadFile(filepath.Join(dir, "aside.json"))
|
|
_ = os.WriteFile(filepath.Join(dir, "aside.json"), []byte(strings.Replace(string(raw), `"step": "restored"`, `"step": "removed"`, 1)), 0o600)
|
|
delete(r.b.streams, "app_notes")
|
|
delete(r.b.streams, "KV_app_seen")
|
|
tmp := filepath.Join(r.e.Dir, "points", "app", "notes", ".tmp-act-x")
|
|
_ = os.MkdirAll(tmp, 0o700)
|
|
_ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-cut.json"), []byte(`{"act":"act-cut","kind":"restore","module":"app","kept":"notes","why":"w","caller":"c","since":"2026-11-12T00:00:00Z"}`), 0o600)
|
|
r.s.events = nil
|
|
if left := r.e.Start(context.Background()); len(left) != 0 {
|
|
t.Errorf("unrestored %+v", left)
|
|
}
|
|
if r.b.streams["app_notes"] != "hello" {
|
|
t.Errorf("the aside was not put back: %v", r.b.streams)
|
|
}
|
|
if _, err := os.Stat(tmp); err == nil {
|
|
t.Errorf("the temporary point stays")
|
|
}
|
|
if ev := lastRestored(t, r); ev.Act != "act-cut" || ev.Error != recovery.ErrStopped {
|
|
t.Errorf("restored %+v", ev)
|
|
}
|
|
}
|
|
|
|
// restore-aside puts an aside back under the live names, refused while live objects of those names exist; discard-aside
|
|
// removes the aside's directory and says aside-discarded (§554, §566).
|
|
func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
_, recs, _ := r.e.Asides()
|
|
id := recs[0].ID
|
|
r.s.events = nil
|
|
r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"})
|
|
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "live stream") {
|
|
t.Errorf("restored %+v", ev)
|
|
}
|
|
r.s.events = nil
|
|
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: "../x", Why: "x", Caller: "tools"})
|
|
if len(r.s.named(seat.EventAsideDiscarded)) != 0 {
|
|
t.Errorf("a path outside was discarded")
|
|
}
|
|
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"})
|
|
if ev := r.s.named(seat.EventAsideDiscarded); len(ev) != 1 || ev[0].Body.(seat.AsideDiscardedEvent).ID != id {
|
|
t.Errorf("events %+v", r.s.events)
|
|
}
|
|
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 {
|
|
t.Errorf("asides %+v", kp.Asides)
|
|
}
|
|
}
|
|
|
|
// The hand verb aside sets the live data aside whole and removes it, naming the streams as a list, and restore-aside puts
|
|
// it back once no live name exists (§516, §554, §566, A-123, A-124).
|
|
func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) {
|
|
r := newRig(t)
|
|
// The findings of §561: aside-unrestored while the aside alone holds the data, points-filling past the fill mark.
|
|
fs := newHeldFindings()
|
|
r.e.Findings, r.e.Node, r.e.Settings.FillingMark, r.e.Settings.Cap = fs, "srv", 1, 4096
|
|
ans := r.do(recovery.Act{Kind: recovery.KindAside, Module: "app", Kept: "notes", Why: "look at it", Caller: "tools", Asker: "op"})
|
|
ev := r.s.named(seat.EventSetAside)
|
|
if ans.Accepted == "" || len(ev) != 1 {
|
|
t.Fatalf("answer %+v events %+v", ans, r.s.events)
|
|
}
|
|
sa := ev[0].Body.(seat.SetAsideEvent)
|
|
if sa.Act != ans.Accepted || len(sa.Names) != 2 || sa.Size == 0 || len(r.b.streams) != 0 {
|
|
t.Errorf("set-aside %+v streams %v", sa, r.b.streams)
|
|
}
|
|
if got := r.e.Unrestored(); len(got) != 1 {
|
|
t.Errorf("unrestored %+v", got)
|
|
}
|
|
// kept-points says it, which the tool and the health check read (A-129).
|
|
if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID {
|
|
t.Errorf("kept-points unrestored %+v", kp.Unrestored)
|
|
}
|
|
if !fs.has(recovery.FindingAsideUnrestored) || !fs.has(recovery.FindingPointsFilling) {
|
|
t.Errorf("findings %+v", fs.Held())
|
|
}
|
|
r.s.events = nil
|
|
r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"})
|
|
if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 {
|
|
t.Errorf("streams %v", r.b.streams)
|
|
}
|
|
r.e.Settings.FillingMark = 100
|
|
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "not needed", Caller: "tools"})
|
|
if fs.has(recovery.FindingAsideUnrestored) || fs.has(recovery.FindingPointsFilling) {
|
|
t.Errorf("findings stand after the aside was restored and discarded: %+v", fs.Held())
|
|
}
|
|
}
|
|
|
|
// Said means acknowledged: a restore whose restore-started the bus did not take reads nothing and sets nothing aside, and
|
|
// ends failed (§563).
|
|
func TestARestoreWhoseStartIsNotAcknowledgedTouchesNothing(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.s.sayFail = map[string]error{seat.EventRestoreStarted: errTest}
|
|
r.do(restoreAct("app/notes/2-fp1"))
|
|
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "restore-started") || r.b.streams["app_notes"] != "hello" {
|
|
t.Errorf("restored %+v, streams %v", ev, r.b.streams)
|
|
}
|
|
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 {
|
|
t.Errorf("an aside was taken: %+v", kp.Asides)
|
|
}
|
|
}
|
|
|
|
// stallBus blocks its first Restore until the act's context ends, as a restore under way when the process stops.
|
|
type stallBus struct {
|
|
*fakeBus
|
|
in chan struct{}
|
|
stalled bool
|
|
}
|
|
|
|
func (s *stallBus) Restore(ctx context.Context, file string) error {
|
|
if !s.stalled {
|
|
s.stalled = true
|
|
close(s.in)
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
}
|
|
return s.fakeBus.Restore(ctx, file)
|
|
}
|
|
|
|
// A stop of the process (a move's or a restart's) answers every new call an error naming it, ends the running restore
|
|
// through its failure path, putting the aside back, and ends every accepted act with `stopped`, so its caller asks again
|
|
// (design 47 G §562, A-141).
|
|
func TestAStopEndsTheRunningActThroughItsFailurePathAndStopped(t *testing.T) {
|
|
r := takenRig(t)
|
|
r.b.streams["app_notes"] = "changed since"
|
|
stall := &stallBus{fakeBus: r.b, in: make(chan struct{})}
|
|
r.e.Bus = stall
|
|
ans := r.e.Accept(restoreAct("app/notes/2-fp1"))
|
|
<-stall.in
|
|
// An accepted act a crash left unended is ended by the stop too.
|
|
_ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-0.json"), []byte(`{"act":"act-0","kind":"take","module":"app","kept":"seen","why":"w","caller":"c","since":"2026-11-11T00:00:00Z"}`), 0o600)
|
|
r.e.Stop("a move")
|
|
if r.b.streams["app_notes"] != "changed since" || r.b.streams["KV_app_seen"] != "world" {
|
|
t.Errorf("the aside is not put back: %v", r.b.streams)
|
|
}
|
|
if ev := lastRestored(t, r); ev.Act != ans.Accepted || ev.Outcome != "failed" || ev.Error != recovery.ErrStopped {
|
|
t.Errorf("restored %+v", ev)
|
|
}
|
|
ended := r.s.named(seat.EventEnded)
|
|
if len(ended) != 1 || ended[0].Body.(seat.EndedEvent).Act != "act-0" || ended[0].Body.(seat.EndedEvent).Error != recovery.ErrStopped {
|
|
t.Errorf("ended %+v", ended)
|
|
}
|
|
if got := r.e.Accept(takeAct()); !strings.Contains(got.Error, "stopping (a move)") {
|
|
t.Errorf("a call during the stop answered %+v", got)
|
|
}
|
|
verb := r.e.Handlers()[seat.VerbKeptPoints]
|
|
if got, _ := verb("c", []byte(`{}`)); !strings.Contains(got.(recovery.Answer).Error, "stopping") {
|
|
t.Errorf("kept-points during the stop answered %+v", got)
|
|
}
|
|
}
|