Files
mesh-catalog/modules/mesh-recovery/internal/recovery/control.go
T
jschoubben 1ed0dca7df mesh-recovery: an act cut off by the process's stop ends stopped
A move or a restart stops the process: from its stop signal every call
is answered an error naming the stop, the running act ends through its
own failure path (an aside put back whole, whatever its context), and
every accepted act not ended ends failed with stopped and its event, so
its caller asks again.

Builds: design 47 G §562, A-141

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 02:15:38 +01:00

190 lines
6.3 KiB
Go

package recovery
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
)
// Cancel is the verb `cancel` (§566, §682): it ends an accepted act that has not begun its work (one waiting on the
// dump, §563) with its ending event, outcome failed and error cancelled, the hand act's words as `why` (A-117); it
// refuses an act that has begun, naming its step, since a restore cut off after its removal is put back only by its own
// failure path (§516). It is written in the hand-act log first and answers in §691's one form.
func (e *Engine) Cancel(act, why, caller, asker string) Answer {
if stop := e.Stopping(); stop != "" {
return Answer{Error: stop}
}
if err := e.Seats.HandAct(caller, asker, "cancel "+act, why); err != nil {
return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()}
}
e.mu.Lock()
defer e.mu.Unlock()
switch {
case e.running == nil || e.running.ID != act:
return Answer{Error: "the act " + act + " is not running"}
case e.running.Step != "waiting":
return Answer{Error: fmt.Sprintf("the act %s has begun (its step: %q) and is put back only by its own failure path", act, e.running.Step)}
}
e.why = why
e.cancel()
return Answer{Accepted: act}
}
// Retry is the verb `retry` (§682): it asks again the last failed act of a module and kept data, with its fields, under
// an act of its own; `nothing to retry` when its last act did not fail.
func (e *Engine) Retry(module, kept, why, caller, asker string) Answer {
acts := e.readActs()
var last *Act
for i := range acts {
a := acts[i]
if a.Module == module && a.Kept == kept && a.Ended && a.Kind != KindExpire && (last == nil || !a.Since.Before(last.Since)) {
last = &a
}
}
if last == nil || last.Error == "" {
return Answer{Error: "nothing to retry"}
}
again := *last
again.ID, again.Ended, again.Error, again.Step, again.WaitsOn = "", false, "", "", ""
again.Why, again.Caller, again.Asker = why, caller, asker
return e.Accept(again)
}
func (e *Engine) readActs() []Act {
var out []Act
files, _ := filepath.Glob(filepath.Join(e.Dir, "acts", "*.json"))
for _, f := range files {
var a Act
if readJSON(f, &a) == nil {
out = append(out, a)
}
}
return out
}
// KeptPoints is the verb `kept-points` (§517): every restore point and aside with their sizes, narrowed to a module and a
// kept data, and the act that runs, with since when and what it waits on.
func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) {
pts, err := e.Points()
if err != nil {
return seat.KeptPointsAnswer{}, err
}
out := seat.KeptPointsAnswer{Points: []seat.Point{}, Asides: []seat.Aside{}}
for _, p := range pts {
if (module == "" || p.Module == module) && (kept == "" || p.Kept == kept) {
out.Points = append(out.Points, p)
}
}
_, recs, _ := e.Asides()
for _, r := range recs {
if (module == "" || r.Module == module) && (kept == "" || r.Kept == kept) {
out.Asides = append(out.Asides, r.aside())
}
}
e.mu.Lock()
if e.running != nil {
out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn}
}
e.mu.Unlock()
// The asides only an aside holds, which the health check fails on, read from the engine (A-129).
out.Unrestored = []seat.Aside{}
for _, x := range e.Unrestored() {
if (module == "" || x.Module == module) && (kept == "" || x.Kept == kept) {
out.Unrestored = append(out.Unrestored, x.Aside)
}
}
return out, nil
}
// Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data
// of those names is gone and only the aside holds it.
func (e *Engine) Unrestored() []UnrestoredAside {
_, recs, _ := e.Asides()
e.mu.Lock()
defer e.mu.Unlock()
var out []UnrestoredAside
for _, r := range recs {
held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept
if r.Step == stepRemoved && !held {
out = append(out, UnrestoredAside{Module: r.Module, Kept: r.Kept, Aside: r.aside()})
}
}
return out
}
// UnrestoredAside is an aside only it holds the data of, with the module and kept data it was taken of: the scope of the
// finding aside-unrestored (§561).
type UnrestoredAside struct {
Module, Kept string
Aside seat.Aside
}
// Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose
// record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under
// the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It
// answers the asides it could not put back (the finding aside-unrestored, §561).
func (e *Engine) Start(ctx context.Context) []UnrestoredAside {
_ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error {
if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") {
_ = os.RemoveAll(path)
return filepath.SkipDir
}
return nil
})
// A keeper's snapshot a cut-off act left half or whole received is deleted (§681).
_ = os.RemoveAll(e.incoming())
_, recs, _ := e.Asides()
for i := range recs {
r := &recs[i]
switch r.Step {
case stepSetAside:
_ = os.RemoveAll(e.asideDirOf(r))
case stepRemoved:
if live, err := e.Bus.Objects(ctx, r.Streams); err == nil && !anyExists(live) {
if err := e.restoreFromAside(ctx, r, false); err != nil {
e.logf("the aside %s is not put back: %v", r.ID, err)
}
}
}
}
e.endUnended()
e.judge()
return e.Unrestored()
}
// endUnended ends every accepted act that never ended failed with `stopped` and its ending event (§554, §562), oldest first.
func (e *Engine) endUnended() {
acts := e.readActs()
sort.Slice(acts, func(i, j int) bool { return acts[i].Since.Before(acts[j].Since) })
for i := range acts {
a := acts[i]
if a.Ended {
continue
}
stopped := errors.New(ErrStopped)
switch a.Kind {
case KindTake:
_ = e.endedFailed(&a, stopped)
case KindRestore, KindRestoreAside:
_ = e.endedRestored(&a, stopped, nil, false)
}
a.Error = ErrStopped
e.markEnded(&a)
}
}
func anyExists(os []Object) bool {
for _, o := range os {
if o.Exists {
return true
}
}
return false
}