Files
mesh-catalog/modules/mesh-recovery/internal/recovery/keeper.go
T
jschoubben bf7d040489 mesh-recovery A-143: the dump wait and the snapshot-restored wait are bound keys
Each wait's bound is a key of the module's state bounds with its own time to
live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's
watch, so no timer stays in the process.

Builds: design 47 G A-143, A-77, A-138, §681, §561

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 02:24:44 +01:00

221 lines
9.3 KiB
Go

package recovery
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
)
// Keeper is the bus module's keeper as the seat mesh-broker answers it (design 47 G §681): its snapshot taken before a
// `drop` or `recreate`, sent by snapshot-get, or restored in place by snapshot-restore when it holds a protected stream.
type Keeper interface {
// Get asks snapshot-get of the request; when its answer names a file and fits accepts it, it receives the chunks into
// w. A refusal of the keeper is the answer's Error, no error of Get.
Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error)
// Restore asks snapshot-restore; with verify the keeper checks the file and restores nothing.
Restore(ctx context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error)
// CancelRestore asks snapshot-restore-cancel of a running restore.
CancelRestore(ctx context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error)
// Restored answers the keeper's snapshot-restored of that request and act once heard (read as §556 has it, so one said
// before the wait counts), or ok false when ctx ends first.
Restored(ctx context.Context, request, act string) (ev meshbroker.SnapshotRestoredEvent, ok bool)
}
// The keeper's word for a snapshot it never hands out (§681).
const snapshotProtected = "snapshot protected"
// ErrUnanswered is the end of a restore in place the keeper answered neither by its outcome nor by the cancel's (§681).
const ErrUnanswered = "snapshot-restore unanswered"
// DefaultRestoreWait is how long a restore in place is waited on: past the keeper's own 60 minutes (§681, A-138).
const DefaultRestoreWait = 90 * time.Minute
func (e *Engine) restoreWait() time.Duration {
if e.Settings.RestoreWait > 0 {
return e.Settings.RestoreWait
}
return DefaultRestoreWait
}
func (e *Engine) incoming() string { return filepath.Join(e.Dir, "incoming") }
// restoreRequest is the hand verb `restore` with `request`, a keeper's snapshot (§681 Its use, §682): an unprotected one is
// read through snapshot-get into incoming/<request>, refused before the first chunk when it would pass the cap, checked
// against its digest before any aside, and restored by the module itself as a point is (§516), the file deleted when the
// act ends; a protected one is restored in place by the keeper (restoreInPlace).
func (e *Engine) restoreRequest(ctx context.Context, a *Act) error {
if e.Keeper == nil {
return e.endedRestored(a, errors.New("no keeper is reachable, so no snapshot of it is restored"), nil, false)
}
e.setStep(a, "receiving the keeper's snapshot", "")
name, err := token(a.Request)
if err != nil {
return e.endedRestored(a, err, nil, false)
}
if err := os.MkdirAll(e.incoming(), 0o700); err != nil {
return e.endedRestored(a, err, nil, false)
}
file := filepath.Join(e.incoming(), name)
defer os.Remove(file)
f, err := os.Create(file)
if err != nil {
return e.endedRestored(a, err, nil, false)
}
ans, err := e.Keeper.Get(ctx, a.Request, func(ans meshbroker.SnapshotGetAnswer) error {
if fill := e.filled(); fill+ans.Size > e.Settings.Cap {
return fmt.Errorf("%s: the cap is %d bytes, the snapshot of the request %s needs %d and the directory holds %d",
ErrCap, e.Settings.Cap, a.Request, ans.Size, fill)
}
return nil
}, f)
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return e.endedRestored(a, err, nil, false)
}
if ans.Error == snapshotProtected {
_ = os.Remove(file)
return e.restoreInPlace(ctx, a)
}
if ans.Error != "" {
return e.endedRestored(a, errors.New(ans.Error), nil, false)
}
// The bytes are checked against the keeper's digest before any aside is taken (§567).
if got, n, err := digestOf(file); err != nil || got != ans.Digest || n != ans.Size {
return e.endedRestored(a, fmt.Errorf("snapshot damaged: the request %s's bytes received do not match its digest", a.Request), nil, false)
}
rec, err := e.setAside(ctx, a, []string{ans.Stream}, true)
if err != nil {
return e.endedRestored(a, err, nil, false)
}
e.setStep(a, "restoring", "")
if err := e.Bus.Restore(ctx, file); err != nil {
err = fmt.Errorf("stream %s: %w", ans.Stream, err)
if back := e.restoreFromAside(ctx, rec, true); back != nil {
err = fmt.Errorf("%w; and the aside is not put back: %v", err, back)
}
return e.endedRestored(a, err, asidePtr(rec), false)
}
rec.Step = stepRestored
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
return e.endedRestored(a, nil, asidePtr(rec), false)
}
// restoreInPlace is a protected snapshot's restore (§681, O-7): snapshot-restore with verify first, then the live data of
// the restore's module and kept data set aside, its names from the catalog entry of the fingerprint the node runs
// (A-137), then snapshot-restore, waited on by snapshot-restored of that request and the act it was answered for at most
// snapshot-restore-wait-minutes (A-138). Its `failed` puts the aside back. At the bound it asks snapshot-restore-cancel and
// waits one more bound for that act's outcome (A-140); when none comes it puts nothing back and ends failed with
// `snapshot-restore unanswered`, the aside left at `removed` for aside-unrestored (§561). A keeper answering busy ends the
// act failed before any aside, naming the keeper's running act (A-140).
func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error {
e.setStep(a, "verifying the keeper's snapshot", "")
v, err := e.Keeper.Restore(ctx, a.Request, true, a.Why)
if err == nil {
err = keeperAnswer(v)
}
if err != nil {
return e.endedRestored(a, err, nil, false)
}
fp, err := e.Seats.RunningFingerprint(ctx, a.Module)
if err != nil {
return e.endedRestored(a, err, nil, false)
}
a.Fingerprint = fp
names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp)
if err != nil {
return e.endedRestored(a, err, nil, false)
}
rec, err := e.setAside(ctx, a, names, true)
if err != nil {
return e.endedRestored(a, err, nil, false)
}
putBack := func(why error) error {
if back := e.restoreFromAside(ctx, rec, true); back != nil {
why = fmt.Errorf("%w; and the aside is not put back: %v", why, back)
}
return e.endedRestored(a, why, asidePtr(rec), false)
}
e.setStep(a, "restoring in place", "the keeper's snapshot-restored")
// The wait's bound is the key `restore-wait.<act>` of the state bounds, written before the keeper is asked, so a bound
// that cannot be written asks nothing of the keeper (A-143).
wctx, passed, err := e.restoreBound(ctx, a.ID)
if err != nil {
return putBack(err)
}
r, err := e.Keeper.Restore(ctx, a.Request, false, a.Why)
if err == nil {
err = keeperAnswer(r)
}
if err != nil {
passed()
return putBack(err)
}
act := r.Accepted
ev, ok := e.Keeper.Restored(wctx, a.Request, act)
passed()
if !ok {
e.setStep(a, "cancelling the keeper's restore", "the keeper's snapshot-restored after the cancel")
why := fmt.Sprintf("the recovery module's wait of %s on the restore in place passed (act %s)", e.restoreWait(), a.ID)
if c, err := e.Keeper.CancelRestore(context.Background(), act, why); err != nil || c.Error != "" {
e.logf("the cancel of the keeper's restore %s is not accepted (%v%s); its outcome is waited on once more", act, err, c.Error)
}
wctx, passed, err := e.restoreBound(context.Background(), a.ID)
if err != nil {
e.logf("the bound of the wait after the cancel of %s is not written: %v", act, err)
return e.endedRestored(a, fmt.Errorf("%s: %v", ErrUnanswered, err), asidePtr(rec), false)
}
ev, ok = e.Keeper.Restored(wctx, a.Request, act)
passed()
}
if !ok {
return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false)
}
if ev.Outcome != meshbroker.SnapshotRestoredEventOutcomeDone {
return putBack(fmt.Errorf("the keeper's restore in place failed: %s", ev.Error))
}
rec.Step = stepRestored
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
return e.endedRestored(a, nil, asidePtr(rec), false)
}
// restoreBound is the bound of one wait on snapshot-restored (§681, A-138, A-143): the key `restore-wait.<act>` with the time
// to live snapshot-restore-wait-minutes; its expiry ends the context it answers. end removes the key.
func (e *Engine) restoreBound(ctx context.Context, act string) (context.Context, func(), error) {
wctx, cancel := context.WithCancel(ctx)
end, err := e.bound(BoundRestoreWait+"."+act, e.restoreWait(), cancel)
if err != nil {
cancel()
return nil, nil, err
}
return wctx, func() { end(); cancel() }, nil
}
// bound writes a bound key in the state bounds (A-77, A-143).
func (e *Engine) bound(key string, ttl time.Duration, expired func()) (func(), error) {
if e.Bounds == nil {
return nil, fmt.Errorf("the bound %s is not written: the engine has no state bounds", key)
}
return e.Bounds.Bound(key, ttl, expired)
}
// keeperAnswer reads snapshot-restore's one form (§691): accepted, or an error naming the keeper's refusal or its running act.
func keeperAnswer(r meshbroker.SnapshotRestoreAnswer) error {
switch {
case r.Error != "":
return errors.New(r.Error)
case r.Busy != "":
return fmt.Errorf("busy: the keeper restores its act %s", r.Busy)
case r.Accepted == "":
return errors.New("the keeper answered snapshot-restore with nothing")
}
return nil
}