Each wait's bound is a key of the module's state bounds with its own time to live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's watch, so no timer stays in the process. Builds: design 47 G A-143, A-77, A-138, §681, §561 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
303 lines
7.1 KiB
Go
303 lines
7.1 KiB
Go
package recovery_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
|
|
"mesh-recovery/internal/recovery"
|
|
)
|
|
|
|
// fakeBus holds streams as strings; a snapshot file is "snap:<name>:<content>".
|
|
type fakeBus struct {
|
|
mu sync.Mutex
|
|
streams map[string]string
|
|
consumers map[string][]seat.Consumer
|
|
setBack []seat.Consumer
|
|
failRestores int
|
|
}
|
|
|
|
func (b *fakeBus) Objects(_ context.Context, names []string) ([]recovery.Object, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
var out []recovery.Object
|
|
for _, n := range names {
|
|
c, ok := b.streams[n]
|
|
out = append(out, recovery.Object{Stream: n, Exists: ok, Bytes: int64(len(c)), Consumers: b.consumers[n]})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (b *fakeBus) Take(_ context.Context, s string, w io.Writer) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
c, ok := b.streams[s]
|
|
if !ok {
|
|
return errors.New("no stream " + s)
|
|
}
|
|
_, err := io.WriteString(w, "snap:"+s+":"+c)
|
|
return err
|
|
}
|
|
|
|
func (b *fakeBus) Delete(_ context.Context, s string) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
delete(b.streams, s)
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Restore(_ context.Context, file string) error {
|
|
raw, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if b.failRestores > 0 {
|
|
b.failRestores--
|
|
return errors.New("the server refused the archive")
|
|
}
|
|
var name, content string
|
|
rest := string(raw[5:])
|
|
for i := range rest {
|
|
if rest[i] == ':' {
|
|
name, content = rest[:i], rest[i+1:]
|
|
break
|
|
}
|
|
}
|
|
if _, live := b.streams[name]; live {
|
|
return errors.New("stream name already in use")
|
|
}
|
|
b.streams[name] = content
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) SetConsumers(_ context.Context, cs []seat.Consumer) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.setBack = append(b.setBack, cs...)
|
|
return nil
|
|
}
|
|
|
|
type said struct {
|
|
Event string
|
|
Body any
|
|
Parts []string
|
|
}
|
|
|
|
type fakeSeats struct {
|
|
mu sync.Mutex
|
|
events []said
|
|
handActs []string
|
|
handFail error
|
|
streams []string
|
|
backup string
|
|
sayFail map[string]error
|
|
}
|
|
|
|
func (s *fakeSeats) Say(event string, body any, parts ...string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if err := s.sayFail[event]; err != nil {
|
|
return err
|
|
}
|
|
s.events = append(s.events, said{event, body, parts})
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeSeats) HandAct(caller, asker, what, why string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.handFail != nil {
|
|
return s.handFail
|
|
}
|
|
s.handActs = append(s.handActs, caller+"|"+asker+"|"+what+"|"+why)
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeSeats) Streams(context.Context, string, string, string) ([]string, error) {
|
|
return s.streams, nil
|
|
}
|
|
|
|
func (s *fakeSeats) RunningFingerprint(context.Context, string) (string, error) { return "fp-run", nil }
|
|
|
|
func (s *fakeSeats) Backup(context.Context, string) (string, error) {
|
|
if s.backup == "" {
|
|
return "", errors.New("no repository snapshot holds it")
|
|
}
|
|
return s.backup, nil
|
|
}
|
|
|
|
func (s *fakeSeats) named(event string) []said {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var out []said
|
|
for _, e := range s.events {
|
|
if e.Event == event {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
type memDigests struct {
|
|
mu sync.Mutex
|
|
m map[string][]byte
|
|
}
|
|
|
|
func (d *memDigests) Put(k string, v []byte) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.m[k] = v
|
|
return nil
|
|
}
|
|
func (d *memDigests) Get(k string) ([]byte, bool, error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
v, ok := d.m[k]
|
|
return v, ok, nil
|
|
}
|
|
func (d *memDigests) Delete(k string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
delete(d.m, k)
|
|
return nil
|
|
}
|
|
|
|
type noDump struct{}
|
|
|
|
func (noDump) Wait(context.Context, func(string)) error { return nil }
|
|
|
|
// holdDump waits until ctx ends, saying it waits.
|
|
type holdDump struct{ waiting chan string }
|
|
|
|
func (h holdDump) Wait(ctx context.Context, waiting func(string)) error {
|
|
waiting("the dump of the bus")
|
|
h.waiting <- "waiting"
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
}
|
|
|
|
type rig struct {
|
|
e *recovery.Engine
|
|
b *fakeBus
|
|
s *fakeSeats
|
|
d *memDigests
|
|
t time.Time
|
|
// bounds is the state bounds the act's waits are timed by (A-143).
|
|
bounds *fakeBounds
|
|
}
|
|
|
|
func newRig(t *testing.T) *rig {
|
|
t.Helper()
|
|
r := &rig{bounds: &fakeBounds{}, b: &fakeBus{streams: map[string]string{"app_notes": "hello", "KV_app_seen": "world"}, consumers: map[string][]seat.Consumer{}},
|
|
s: &fakeSeats{streams: []string{"app_notes", "KV_app_seen"}}, d: &memDigests{m: map[string][]byte{}}, t: time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)}
|
|
r.b.consumers["app_notes"] = []seat.Consumer{{Name: "c1", Stream: "app_notes", Sequence: 7}}
|
|
r.e = &recovery.Engine{Dir: filepath.Join(t.TempDir(), "points-dir"), Bus: r.b, Seats: r.s, Digests: r.d, Dump: noDump{}, Bounds: r.bounds,
|
|
Settings: recovery.Settings{Cap: 1 << 20, KeepDays: 90}, Now: func() time.Time { return r.t }}
|
|
return r
|
|
}
|
|
|
|
func (r *rig) do(a recovery.Act) recovery.Answer {
|
|
ans := r.e.Accept(a)
|
|
r.e.Wait()
|
|
return ans
|
|
}
|
|
|
|
var errTest = errors.New("the log is unreachable")
|
|
|
|
type gatedBus struct {
|
|
*fakeBus
|
|
gate chan struct{}
|
|
}
|
|
|
|
func (g *gatedBus) Objects(ctx context.Context, n []string) ([]recovery.Object, error) {
|
|
<-g.gate
|
|
return g.fakeBus.Objects(ctx, n)
|
|
}
|
|
|
|
// heldFindings is the module's finding writer as it holds them; written says each write.
|
|
type heldFindings struct {
|
|
mu sync.Mutex
|
|
held map[string]meshconditions.Finding
|
|
written chan meshconditions.Finding
|
|
}
|
|
|
|
func newHeldFindings() *heldFindings {
|
|
return &heldFindings{held: map[string]meshconditions.Finding{}, written: make(chan meshconditions.Finding, 64)}
|
|
}
|
|
|
|
func (h *heldFindings) Write(_ context.Context, f meshconditions.Finding) error {
|
|
h.mu.Lock()
|
|
h.held[f.Scope+"."+f.ID+"."+f.Kind] = f
|
|
h.mu.Unlock()
|
|
h.written <- f
|
|
return nil
|
|
}
|
|
|
|
func (h *heldFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
delete(h.held, scope+"."+id+"."+kind)
|
|
return nil
|
|
}
|
|
|
|
func (h *heldFindings) Held() []meshconditions.Finding {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
var out []meshconditions.Finding
|
|
for _, f := range h.held {
|
|
out = append(out, f)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (h *heldFindings) has(kind string) bool {
|
|
for _, f := range h.Held() {
|
|
if f.Kind == kind {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// fakeBounds is the state bounds: each key removed by the bus when its time to live passes, as the bus does, or by its
|
|
// end; written and removed say which keys, with each key's time to live (A-143).
|
|
type fakeBounds struct {
|
|
mu sync.Mutex
|
|
written []string
|
|
removed []string
|
|
}
|
|
|
|
func (f *fakeBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) {
|
|
f.mu.Lock()
|
|
f.written = append(f.written, key+" "+ttl.String())
|
|
f.mu.Unlock()
|
|
var once sync.Once
|
|
gone := func(by string, then func()) {
|
|
once.Do(func() {
|
|
f.mu.Lock()
|
|
f.removed = append(f.removed, key+" "+by)
|
|
f.mu.Unlock()
|
|
then()
|
|
})
|
|
}
|
|
t := time.AfterFunc(ttl, func() { gone("expired", expired) })
|
|
return func() { t.Stop(); gone("ended", func() {}) }, nil
|
|
}
|
|
|
|
func (f *fakeBounds) keys() (written, removed string) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return strings.Join(f.written, "|"), strings.Join(f.removed, "|")
|
|
}
|