Files
mesh-host/internal/agent/kept_test.go
T
jschoubben d217056cf7 H-A-177: move a data module without its restore point where no recovery holder is placed
The node-engine held every data-holding module's take-in for good: it asked
the seat mesh-recovery for the restore point, and nothing answers because
no holder of the seat is placed on the mesh (H §456). It now asks only where
the assigner's placements it holds name a holder (and while they are not
read yet); where none is placed it moves the module without a point and
raises the finding restore-point-missing (warning, scope machine, id the
module), cleared at the module's next take-in that takes its point. A holder
placed that does not answer still holds the move.

Builds: design 47 H §456, H-A-177
2026-11-20 18:56:43 +01:00

380 lines
16 KiB
Go

package agent
import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
"github.com/novox/mesh-host/internal/compositions"
)
// A recovery module that answers as scripted: each ask takes the next answer and says the events it carries after it.
type recoveryFake struct {
mu sync.Mutex
asks []string // verb and its input, in order
answers []fakeAnswer
points []meshrecovery.Point
events chan Ending
changes []catalogue.FormChange
calls []string // stop, ask, apply, catch-up, in order
catchUp []meshrecovery.CatchUp
expired chan struct{}
unknown error // FormChanges answers it
waited int // times the agent told the composer it waits for a composition
// noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177).
noHolder, unread bool
missing []string // the finding restore-point-missing raised ("raise: <words>") or cleared ("clear")
}
type fakeAnswer struct {
accepted, busy, errWords string
then []Ending
}
func (f *recoveryFake) next(verb string, in any) (string, string, string) {
f.mu.Lock()
defer f.mu.Unlock()
f.asks = append(f.asks, fmt.Sprintf("%s %s", verb, asJSON(in)))
f.calls = append(f.calls, verb)
if len(f.answers) == 0 {
return "", "", "no answer scripted"
}
a := f.answers[0]
f.answers = f.answers[1:]
for _, e := range a.then {
f.events <- e
}
return a.accepted, a.busy, a.errWords
}
func (f *recoveryFake) env() *Recovery {
return &Recovery{
FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) {
return f.changes, f.unknown
},
SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
a, b, e := f.next("snapshot-kept", in)
return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
},
RestoreKept: func(_ context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error) {
a, b, e := f.next("restore-kept", in)
return meshrecovery.RestoreKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
},
KeptPoints: func(context.Context, meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, "kept-points")
return meshrecovery.KeptPointsAnswer{Points: f.points}, nil
},
Hear: func(context.Context, string, string, string) (<-chan Ending, func(), error) {
return f.events, func() {}, nil
},
Bound: func(context.Context, string, string, string, time.Time, time.Duration) (<-chan struct{}, func(), error) {
return f.expired, func() {}, nil
},
CatchUp: func(_ context.Context, _ string, c meshrecovery.CatchUp) error {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, "catch-up")
f.catchUp = append(f.catchUp, c)
return nil
},
Placed: func() (bool, bool) { return !f.noHolder, !f.unread },
PointMissing: func(_ context.Context, module, words string, missing bool) {
f.mu.Lock()
defer f.mu.Unlock()
if missing {
f.missing = append(f.missing, "raise "+module+": "+words)
} else {
f.missing = append(f.missing, "clear "+module)
}
},
Now: time.Now,
}
}
func (f *recoveryFake) seen() []string {
f.mu.Lock()
defer f.mu.Unlock()
return slices.Clone(f.calls)
}
// moveDB runs a data module at fp "a", then wants it at fp "b", and answers the bed once the second want is settled or held.
func moveDB(t *testing.T, f *recoveryFake, ranFrom, to string, settle func(b *bed) bool, before ...func(b *bed, dig string)) *bed {
t.Helper()
b := newBed(t)
inner, applies := b.env.Apply, 0
b.env.Apply = func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result {
if applies++; applies >= 2 {
f.mu.Lock()
f.calls = append(f.calls, "apply")
f.mu.Unlock()
}
return inner(ctx, c, others)
}
f.events, f.expired = make(chan Ending, 16), make(chan struct{})
b.env.Recovery = f.env()
b.env.Waiting = func(string, string) { f.mu.Lock(); f.waited++; f.mu.Unlock() }
b.env.Stop = func(context.Context, string) Result {
f.mu.Lock()
f.calls = append(f.calls, "stop")
f.mu.Unlock()
return Result{}
}
stop := b.run()
t.Cleanup(stop)
fp1, fp2 := fpOf(ranFrom), fpOf(to)
ck, craw, dig := composition(t, "db", fp1, true, file("db", "x", "1"))
wk, wraw := wantFor(t, "db", fp1, dig, "")
b.comps.put(ck, craw)
b.wants.put(wk, wraw)
eventually(t, "the first build is not applied", func() bool { return len(b.appliedList()) == 1 })
b.idle()
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
for _, h := range before {
h(b, dig)
}
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
b.comps.put(ck2, craw2)
b.wants.put(wk2, wraw2)
eventually(t, "the move is neither applied nor held", func() bool { return settle(b) })
return b
}
// testAMoveAcrossAFormChange is design 47 G §296, §297, §554, §560, §630 to §632 and A-98, A-108 at the node agent.
func testAMoveAcrossAFormChange(t *testing.T) {
up := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1},
Applied: &catalogue.KeptLayout{Form: 2, CatchUp: &catalogue.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}}
down := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 2},
Applied: &catalogue.KeptLayout{Form: 1}}}
fp := func(c string) string { return fpOf(c) }
done := func(act string) Ending { return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeDone} }
failed := func(act, why string) Ending {
e := Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why}
if why == "cancelled" {
e.Why = "the operator stopped it" // the hand act's words (design 47 G A-117)
}
return e
}
applied := func(b *bed) bool { return len(b.appliedList()) == 2 }
heldHave := func(b *bed) bool { return b.haves.last("db").Phase == "held" }
t.Run("a form that rises is taken as a restore point before the move, with the why and the catch-up, and then applied", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{done("a1")}}}}
moveDB(t, f, "a", "b", applied)
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
want := fmt.Sprintf("snapshot-kept %s", asJSON(meshrecovery.SnapshotKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("b"),
Why: "form change of db's cache to " + fp("b"), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}))
if f.asks[0] != want {
t.Fatalf("asked %s, want %s", f.asks[0], want)
}
if !slices.Equal(f.missing, []string{"clear db"}) {
t.Fatalf("a take-in with its point did not clear restore-point-missing: %v", f.missing)
}
})
// Design 47 H, H-A-177, H §456: the point is asked where the placements name a holder of mesh-recovery (and while they are not
// read yet); where none is placed on the mesh the module moves without one, raising restore-point-missing; a holder placed that
// does not answer holds the move.
for _, row := range []struct {
name string
noHolder, unread bool
calls []string
moved bool
missing string
}{
{name: "no holder of mesh-recovery placed on the mesh moves the module without its point and raises restore-point-missing",
noHolder: true, calls: []string{"apply"}, moved: true, missing: "raise db: its kept data cache (form 1) moved to " + fp("b") + " with no restore point taken: no holder of the seat mesh-recovery is placed on the mesh"},
{name: "a holder placed that does not answer holds the move and raises nothing", calls: []string{"snapshot-kept"}},
{name: "placements not read yet ask the point as before", noHolder: true, unread: true, calls: []string{"snapshot-kept"}},
} {
t.Run(row.name, func(t *testing.T) {
f := &recoveryFake{changes: up, noHolder: row.noHolder, unread: row.unread}
settle := heldHave
if row.moved {
settle = applied
}
b := moveDB(t, f, "a", "b", settle)
if got := f.seen(); !slices.Equal(got, row.calls) {
t.Fatalf("calls %v, want %v", got, row.calls)
}
f.mu.Lock()
missing := slices.Clone(f.missing)
f.mu.Unlock()
if row.missing == "" && len(missing) != 0 || row.missing != "" && !slices.Equal(missing, []string{row.missing}) {
t.Fatalf("restore-point-missing %v, want %q", missing, row.missing)
}
if h := b.haves.last("db"); row.moved && h.Phase == "held" || !row.moved && h.FP != fp("a") {
t.Fatalf("the have %+v", h)
}
})
}
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
{accepted: "a2", then: []Ending{failed("a2", "stopped")}},
{accepted: "a3", then: []Ending{done("a3")}},
}}
moveDB(t, f, "a", "b", applied)
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "snapshot-kept", "snapshot-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
})
for _, row := range []struct{ error, held string }{
{"cancelled", "cancelled: the operator stopped it"},
{"cap", "cap"},
{"restore point damaged", "restore point damaged"},
{"disk full", "disk full"},
} {
t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" {
t.Fatalf("the have %+v", h)
}
if got := f.seen(); len(got) != 1 {
t.Fatalf("asked again: %v", got)
}
if n := len(b.appliedList()); n != 1 {
t.Fatalf("applied %d", n)
}
})
}
t.Run("the bound passed says the wait in the have and the wait goes on to the ending", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1"}}}
b := moveDB(t, f, "a", "b", func(b *bed) bool {
b.haves.mu.Lock()
defer b.haves.mu.Unlock()
for _, h := range b.haves.log {
if h.Wait != nil && h.Wait.Words != "" && h.Phase == "backing-up" {
return true
}
}
return false
}, func(*bed, string) { close(f.expired) })
f.events <- done("a1")
eventually(t, "the move went on after the ending", func() bool { return applied(b) })
})
t.Run("a revert stops the newer build, restores the point of the form it reverts from, applies, then calls its catch-up", func(t *testing.T) {
f := &recoveryFake{changes: down,
points: []meshrecovery.Point{
{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("z"), Snapshot: "other", At: time.Now()},
{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now(), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild"}}},
answers: []fakeAnswer{{accepted: "r1", then: []Ending{done("r1")}}}}
moveDB(t, f, "a", "b", func(b *bed) bool { return len(f.seen()) >= 5 })
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply", "catch-up"}) {
t.Fatalf("calls %v", got)
}
if f.catchUp[0].Tool != "rebuild" {
t.Fatalf("catch-up %+v", f.catchUp)
}
want := fmt.Sprintf("restore-kept %s", asJSON(meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"),
Snapshot: "snap-1", Why: "revert of db's cache from " + fp("a")}))
if f.asks[0] != want {
t.Fatalf("asked %s, want %s", f.asks[0], want)
}
})
t.Run("a revert with no point of its form holds as a restore point missing", func(t *testing.T) {
f := &recoveryFake{changes: down}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
t.Fatalf("the have %+v", h)
}
})
t.Run("a held revert, then the want with revert live, starts the older build with no restore, no aside and no catch-up (design 47 G section 710)", func(t *testing.T) {
f := &recoveryFake{changes: down}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
t.Fatalf("the have %+v", h)
}
before := len(b.appliedList())
_, _, _ = composition(t, "db", fpOf("b"), true, file("db", "x", "2"))
raw, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fpOf("b"), Revert: meshwants.WantsByWantNodeModuleValueRevertLive,
Words: "the operator answered \"live\" to the revert ask (asking 1)"})
if err != nil {
t.Fatal(err)
}
b.wants.put(meshwants.WantsByWantNodeModuleKey(node, "db"), raw)
eventually(t, "the older build runs on the answer", func() bool { h := b.haves.last("db"); return h.FP == fpOf("b") && h.Phase != "held" })
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "apply", "apply"}) && len(b.appliedList()) <= before {
t.Fatalf("calls %v", got)
}
for _, c := range f.seen() {
if c == "restore-kept" || c == "snapshot-kept" || c == "catch-up" {
t.Fatalf("calls %v", f.seen())
}
}
if n := strings.Count(strings.Join(f.seen(), " "), "kept-points"); n != 1 {
t.Fatalf("the points were read again on the answer: %v", f.seen())
}
})
t.Run("a revert whose restore fails starts the newer build again and holds the move, saying why (A-113)", func(t *testing.T) {
f := &recoveryFake{changes: down,
points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now()}},
answers: []fakeAnswer{{accepted: "r1", then: []Ending{failed("r1", "restore point damaged")}}}}
b := moveDB(t, f, "a", "b", heldHave)
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
if h := b.haves.last("db"); !contains(h.Detail, "restore point damaged") || !contains(h.Detail, "the newer build runs again") ||
h.Wait == nil || h.Wait.Step != nodeengine.WaitStepCompose {
t.Fatalf("the have %+v", h)
}
})
t.Run("a form change not known moves nothing and waits for the entries (A-113)", func(t *testing.T) {
f := &recoveryFake{unknown: fmt.Errorf("the entry is asked")}
b := moveDB(t, f, "a", "b", heldHave)
f.mu.Lock()
if f.waited != 0 {
t.Errorf("the composer was told the agent waits for a composition %d times; it waits for an entry and holds the composition", f.waited)
}
f.mu.Unlock()
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, "is not known") || h.Wait == nil || h.Wait.Reason != "entry-missing" {
t.Fatalf("the have %+v", h)
}
if n := len(b.appliedList()); n != 1 || len(f.seen()) != 0 {
t.Fatalf("applied %d, calls %v", n, f.seen())
}
})
t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) {
f := &recoveryFake{changes: up}
b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) {
at := time.Now().Add(-time.Minute)
b.haves.mu.Lock()
b.haves.stored["db"] = Have{Module: "db", FP: fp("a"), Phase: "backing-up", Wait: &nodeengine.Wait{
On: "mesh-recovery.ended.db.cache", AskAt: &at, Act: "a1", Since: at}}
b.haves.mu.Unlock()
f.events <- done("a1")
})
_ = b
if got := f.seen(); !slices.Equal(got, []string{"apply"}) {
t.Fatalf("calls %v", got)
}
})
}
func contains(s, sub string) bool { return strings.Contains(s, sub) }
func asJSON(v any) string { raw, _ := json.Marshal(v); return string(raw) }