A move or a restart stops the process: from its stop signal every call is answered an error naming the stop, the running act ends through its own failure path (an aside put back whole, whatever its context), and every accepted act not ended ends failed with stopped and its event, so its caller asks again. Builds: design 47 G §562, A-141 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
190 lines
6.3 KiB
Go
190 lines
6.3 KiB
Go
package recovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
|
)
|
|
|
|
// Cancel is the verb `cancel` (§566, §682): it ends an accepted act that has not begun its work (one waiting on the
|
|
// dump, §563) with its ending event, outcome failed and error cancelled, the hand act's words as `why` (A-117); it
|
|
// refuses an act that has begun, naming its step, since a restore cut off after its removal is put back only by its own
|
|
// failure path (§516). It is written in the hand-act log first and answers in §691's one form.
|
|
func (e *Engine) Cancel(act, why, caller, asker string) Answer {
|
|
if stop := e.Stopping(); stop != "" {
|
|
return Answer{Error: stop}
|
|
}
|
|
if err := e.Seats.HandAct(caller, asker, "cancel "+act, why); err != nil {
|
|
return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()}
|
|
}
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
switch {
|
|
case e.running == nil || e.running.ID != act:
|
|
return Answer{Error: "the act " + act + " is not running"}
|
|
case e.running.Step != "waiting":
|
|
return Answer{Error: fmt.Sprintf("the act %s has begun (its step: %q) and is put back only by its own failure path", act, e.running.Step)}
|
|
}
|
|
e.why = why
|
|
e.cancel()
|
|
return Answer{Accepted: act}
|
|
}
|
|
|
|
// Retry is the verb `retry` (§682): it asks again the last failed act of a module and kept data, with its fields, under
|
|
// an act of its own; `nothing to retry` when its last act did not fail.
|
|
func (e *Engine) Retry(module, kept, why, caller, asker string) Answer {
|
|
acts := e.readActs()
|
|
var last *Act
|
|
for i := range acts {
|
|
a := acts[i]
|
|
if a.Module == module && a.Kept == kept && a.Ended && a.Kind != KindExpire && (last == nil || !a.Since.Before(last.Since)) {
|
|
last = &a
|
|
}
|
|
}
|
|
if last == nil || last.Error == "" {
|
|
return Answer{Error: "nothing to retry"}
|
|
}
|
|
again := *last
|
|
again.ID, again.Ended, again.Error, again.Step, again.WaitsOn = "", false, "", "", ""
|
|
again.Why, again.Caller, again.Asker = why, caller, asker
|
|
return e.Accept(again)
|
|
}
|
|
|
|
func (e *Engine) readActs() []Act {
|
|
var out []Act
|
|
files, _ := filepath.Glob(filepath.Join(e.Dir, "acts", "*.json"))
|
|
for _, f := range files {
|
|
var a Act
|
|
if readJSON(f, &a) == nil {
|
|
out = append(out, a)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// KeptPoints is the verb `kept-points` (§517): every restore point and aside with their sizes, narrowed to a module and a
|
|
// kept data, and the act that runs, with since when and what it waits on.
|
|
func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) {
|
|
pts, err := e.Points()
|
|
if err != nil {
|
|
return seat.KeptPointsAnswer{}, err
|
|
}
|
|
out := seat.KeptPointsAnswer{Points: []seat.Point{}, Asides: []seat.Aside{}}
|
|
for _, p := range pts {
|
|
if (module == "" || p.Module == module) && (kept == "" || p.Kept == kept) {
|
|
out.Points = append(out.Points, p)
|
|
}
|
|
}
|
|
_, recs, _ := e.Asides()
|
|
for _, r := range recs {
|
|
if (module == "" || r.Module == module) && (kept == "" || r.Kept == kept) {
|
|
out.Asides = append(out.Asides, r.aside())
|
|
}
|
|
}
|
|
e.mu.Lock()
|
|
if e.running != nil {
|
|
out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn}
|
|
}
|
|
e.mu.Unlock()
|
|
// The asides only an aside holds, which the health check fails on, read from the engine (A-129).
|
|
out.Unrestored = []seat.Aside{}
|
|
for _, x := range e.Unrestored() {
|
|
if (module == "" || x.Module == module) && (kept == "" || x.Kept == kept) {
|
|
out.Unrestored = append(out.Unrestored, x.Aside)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data
|
|
// of those names is gone and only the aside holds it.
|
|
func (e *Engine) Unrestored() []UnrestoredAside {
|
|
_, recs, _ := e.Asides()
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
var out []UnrestoredAside
|
|
for _, r := range recs {
|
|
held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept
|
|
if r.Step == stepRemoved && !held {
|
|
out = append(out, UnrestoredAside{Module: r.Module, Kept: r.Kept, Aside: r.aside()})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// UnrestoredAside is an aside only it holds the data of, with the module and kept data it was taken of: the scope of the
|
|
// finding aside-unrestored (§561).
|
|
type UnrestoredAside struct {
|
|
Module, Kept string
|
|
Aside seat.Aside
|
|
}
|
|
|
|
// Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose
|
|
// record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under
|
|
// the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It
|
|
// answers the asides it could not put back (the finding aside-unrestored, §561).
|
|
func (e *Engine) Start(ctx context.Context) []UnrestoredAside {
|
|
_ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error {
|
|
if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") {
|
|
_ = os.RemoveAll(path)
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
})
|
|
// A keeper's snapshot a cut-off act left half or whole received is deleted (§681).
|
|
_ = os.RemoveAll(e.incoming())
|
|
_, recs, _ := e.Asides()
|
|
for i := range recs {
|
|
r := &recs[i]
|
|
switch r.Step {
|
|
case stepSetAside:
|
|
_ = os.RemoveAll(e.asideDirOf(r))
|
|
case stepRemoved:
|
|
if live, err := e.Bus.Objects(ctx, r.Streams); err == nil && !anyExists(live) {
|
|
if err := e.restoreFromAside(ctx, r, false); err != nil {
|
|
e.logf("the aside %s is not put back: %v", r.ID, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
e.endUnended()
|
|
e.judge()
|
|
return e.Unrestored()
|
|
}
|
|
|
|
// endUnended ends every accepted act that never ended failed with `stopped` and its ending event (§554, §562), oldest first.
|
|
func (e *Engine) endUnended() {
|
|
acts := e.readActs()
|
|
sort.Slice(acts, func(i, j int) bool { return acts[i].Since.Before(acts[j].Since) })
|
|
for i := range acts {
|
|
a := acts[i]
|
|
if a.Ended {
|
|
continue
|
|
}
|
|
stopped := errors.New(ErrStopped)
|
|
switch a.Kind {
|
|
case KindTake:
|
|
_ = e.endedFailed(&a, stopped)
|
|
case KindRestore, KindRestoreAside:
|
|
_ = e.endedRestored(&a, stopped, nil, false)
|
|
}
|
|
a.Error = ErrStopped
|
|
e.markEnded(&a)
|
|
}
|
|
}
|
|
|
|
func anyExists(os []Object) bool {
|
|
for _, o := range os {
|
|
if o.Exists {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|