Each wait's bound is a key of the module's state bounds with its own time to live, dump-wait.<act> and restore-wait.<act>, its expiry heard on the key's watch, so no timer stays in the process. Builds: design 47 G A-143, A-77, A-138, §681, §561 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
221 lines
9.3 KiB
Go
221 lines
9.3 KiB
Go
package recovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
|
)
|
|
|
|
// Keeper is the bus module's keeper as the seat mesh-broker answers it (design 47 G §681): its snapshot taken before a
|
|
// `drop` or `recreate`, sent by snapshot-get, or restored in place by snapshot-restore when it holds a protected stream.
|
|
type Keeper interface {
|
|
// Get asks snapshot-get of the request; when its answer names a file and fits accepts it, it receives the chunks into
|
|
// w. A refusal of the keeper is the answer's Error, no error of Get.
|
|
Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error)
|
|
// Restore asks snapshot-restore; with verify the keeper checks the file and restores nothing.
|
|
Restore(ctx context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error)
|
|
// CancelRestore asks snapshot-restore-cancel of a running restore.
|
|
CancelRestore(ctx context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error)
|
|
// Restored answers the keeper's snapshot-restored of that request and act once heard (read as §556 has it, so one said
|
|
// before the wait counts), or ok false when ctx ends first.
|
|
Restored(ctx context.Context, request, act string) (ev meshbroker.SnapshotRestoredEvent, ok bool)
|
|
}
|
|
|
|
// The keeper's word for a snapshot it never hands out (§681).
|
|
const snapshotProtected = "snapshot protected"
|
|
|
|
// ErrUnanswered is the end of a restore in place the keeper answered neither by its outcome nor by the cancel's (§681).
|
|
const ErrUnanswered = "snapshot-restore unanswered"
|
|
|
|
// DefaultRestoreWait is how long a restore in place is waited on: past the keeper's own 60 minutes (§681, A-138).
|
|
const DefaultRestoreWait = 90 * time.Minute
|
|
|
|
func (e *Engine) restoreWait() time.Duration {
|
|
if e.Settings.RestoreWait > 0 {
|
|
return e.Settings.RestoreWait
|
|
}
|
|
return DefaultRestoreWait
|
|
}
|
|
|
|
func (e *Engine) incoming() string { return filepath.Join(e.Dir, "incoming") }
|
|
|
|
// restoreRequest is the hand verb `restore` with `request`, a keeper's snapshot (§681 Its use, §682): an unprotected one is
|
|
// read through snapshot-get into incoming/<request>, refused before the first chunk when it would pass the cap, checked
|
|
// against its digest before any aside, and restored by the module itself as a point is (§516), the file deleted when the
|
|
// act ends; a protected one is restored in place by the keeper (restoreInPlace).
|
|
func (e *Engine) restoreRequest(ctx context.Context, a *Act) error {
|
|
if e.Keeper == nil {
|
|
return e.endedRestored(a, errors.New("no keeper is reachable, so no snapshot of it is restored"), nil, false)
|
|
}
|
|
e.setStep(a, "receiving the keeper's snapshot", "")
|
|
name, err := token(a.Request)
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
if err := os.MkdirAll(e.incoming(), 0o700); err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
file := filepath.Join(e.incoming(), name)
|
|
defer os.Remove(file)
|
|
f, err := os.Create(file)
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
ans, err := e.Keeper.Get(ctx, a.Request, func(ans meshbroker.SnapshotGetAnswer) error {
|
|
if fill := e.filled(); fill+ans.Size > e.Settings.Cap {
|
|
return fmt.Errorf("%s: the cap is %d bytes, the snapshot of the request %s needs %d and the directory holds %d",
|
|
ErrCap, e.Settings.Cap, a.Request, ans.Size, fill)
|
|
}
|
|
return nil
|
|
}, f)
|
|
if cerr := f.Close(); err == nil {
|
|
err = cerr
|
|
}
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
if ans.Error == snapshotProtected {
|
|
_ = os.Remove(file)
|
|
return e.restoreInPlace(ctx, a)
|
|
}
|
|
if ans.Error != "" {
|
|
return e.endedRestored(a, errors.New(ans.Error), nil, false)
|
|
}
|
|
// The bytes are checked against the keeper's digest before any aside is taken (§567).
|
|
if got, n, err := digestOf(file); err != nil || got != ans.Digest || n != ans.Size {
|
|
return e.endedRestored(a, fmt.Errorf("snapshot damaged: the request %s's bytes received do not match its digest", a.Request), nil, false)
|
|
}
|
|
rec, err := e.setAside(ctx, a, []string{ans.Stream}, true)
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
e.setStep(a, "restoring", "")
|
|
if err := e.Bus.Restore(ctx, file); err != nil {
|
|
err = fmt.Errorf("stream %s: %w", ans.Stream, err)
|
|
if back := e.restoreFromAside(ctx, rec, true); back != nil {
|
|
err = fmt.Errorf("%w; and the aside is not put back: %v", err, back)
|
|
}
|
|
return e.endedRestored(a, err, asidePtr(rec), false)
|
|
}
|
|
rec.Step = stepRestored
|
|
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
|
|
return e.endedRestored(a, nil, asidePtr(rec), false)
|
|
}
|
|
|
|
// restoreInPlace is a protected snapshot's restore (§681, O-7): snapshot-restore with verify first, then the live data of
|
|
// the restore's module and kept data set aside, its names from the catalog entry of the fingerprint the node runs
|
|
// (A-137), then snapshot-restore, waited on by snapshot-restored of that request and the act it was answered for at most
|
|
// snapshot-restore-wait-minutes (A-138). Its `failed` puts the aside back. At the bound it asks snapshot-restore-cancel and
|
|
// waits one more bound for that act's outcome (A-140); when none comes it puts nothing back and ends failed with
|
|
// `snapshot-restore unanswered`, the aside left at `removed` for aside-unrestored (§561). A keeper answering busy ends the
|
|
// act failed before any aside, naming the keeper's running act (A-140).
|
|
func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error {
|
|
e.setStep(a, "verifying the keeper's snapshot", "")
|
|
v, err := e.Keeper.Restore(ctx, a.Request, true, a.Why)
|
|
if err == nil {
|
|
err = keeperAnswer(v)
|
|
}
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
fp, err := e.Seats.RunningFingerprint(ctx, a.Module)
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
a.Fingerprint = fp
|
|
names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp)
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
rec, err := e.setAside(ctx, a, names, true)
|
|
if err != nil {
|
|
return e.endedRestored(a, err, nil, false)
|
|
}
|
|
putBack := func(why error) error {
|
|
if back := e.restoreFromAside(ctx, rec, true); back != nil {
|
|
why = fmt.Errorf("%w; and the aside is not put back: %v", why, back)
|
|
}
|
|
return e.endedRestored(a, why, asidePtr(rec), false)
|
|
}
|
|
e.setStep(a, "restoring in place", "the keeper's snapshot-restored")
|
|
// The wait's bound is the key `restore-wait.<act>` of the state bounds, written before the keeper is asked, so a bound
|
|
// that cannot be written asks nothing of the keeper (A-143).
|
|
wctx, passed, err := e.restoreBound(ctx, a.ID)
|
|
if err != nil {
|
|
return putBack(err)
|
|
}
|
|
r, err := e.Keeper.Restore(ctx, a.Request, false, a.Why)
|
|
if err == nil {
|
|
err = keeperAnswer(r)
|
|
}
|
|
if err != nil {
|
|
passed()
|
|
return putBack(err)
|
|
}
|
|
act := r.Accepted
|
|
ev, ok := e.Keeper.Restored(wctx, a.Request, act)
|
|
passed()
|
|
if !ok {
|
|
e.setStep(a, "cancelling the keeper's restore", "the keeper's snapshot-restored after the cancel")
|
|
why := fmt.Sprintf("the recovery module's wait of %s on the restore in place passed (act %s)", e.restoreWait(), a.ID)
|
|
if c, err := e.Keeper.CancelRestore(context.Background(), act, why); err != nil || c.Error != "" {
|
|
e.logf("the cancel of the keeper's restore %s is not accepted (%v%s); its outcome is waited on once more", act, err, c.Error)
|
|
}
|
|
wctx, passed, err := e.restoreBound(context.Background(), a.ID)
|
|
if err != nil {
|
|
e.logf("the bound of the wait after the cancel of %s is not written: %v", act, err)
|
|
return e.endedRestored(a, fmt.Errorf("%s: %v", ErrUnanswered, err), asidePtr(rec), false)
|
|
}
|
|
ev, ok = e.Keeper.Restored(wctx, a.Request, act)
|
|
passed()
|
|
}
|
|
if !ok {
|
|
return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false)
|
|
}
|
|
if ev.Outcome != meshbroker.SnapshotRestoredEventOutcomeDone {
|
|
return putBack(fmt.Errorf("the keeper's restore in place failed: %s", ev.Error))
|
|
}
|
|
rec.Step = stepRestored
|
|
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
|
|
return e.endedRestored(a, nil, asidePtr(rec), false)
|
|
}
|
|
|
|
// restoreBound is the bound of one wait on snapshot-restored (§681, A-138, A-143): the key `restore-wait.<act>` with the time
|
|
// to live snapshot-restore-wait-minutes; its expiry ends the context it answers. end removes the key.
|
|
func (e *Engine) restoreBound(ctx context.Context, act string) (context.Context, func(), error) {
|
|
wctx, cancel := context.WithCancel(ctx)
|
|
end, err := e.bound(BoundRestoreWait+"."+act, e.restoreWait(), cancel)
|
|
if err != nil {
|
|
cancel()
|
|
return nil, nil, err
|
|
}
|
|
return wctx, func() { end(); cancel() }, nil
|
|
}
|
|
|
|
// bound writes a bound key in the state bounds (A-77, A-143).
|
|
func (e *Engine) bound(key string, ttl time.Duration, expired func()) (func(), error) {
|
|
if e.Bounds == nil {
|
|
return nil, fmt.Errorf("the bound %s is not written: the engine has no state bounds", key)
|
|
}
|
|
return e.Bounds.Bound(key, ttl, expired)
|
|
}
|
|
|
|
// keeperAnswer reads snapshot-restore's one form (§691): accepted, or an error naming the keeper's refusal or its running act.
|
|
func keeperAnswer(r meshbroker.SnapshotRestoreAnswer) error {
|
|
switch {
|
|
case r.Error != "":
|
|
return errors.New(r.Error)
|
|
case r.Busy != "":
|
|
return fmt.Errorf("busy: the keeper restores its act %s", r.Busy)
|
|
case r.Accepted == "":
|
|
return errors.New("the keeper answered snapshot-restore with nothing")
|
|
}
|
|
return nil
|
|
}
|