package keeper import ( "context" "encoding/json" "errors" "fmt" "os" "time" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" ) // The keeper's restore in place of a snapshot it took (design 47 G §681, ADR 0675; the bus module hands the work through // the state requests, A-45): the recovery module asks the seat mesh-broker's snapshot-restore for a protected snapshot, which // is never handed out. The keeper checks the file against its recorded digest, refuses while a live stream of the snapshot's // names exists, and restores them under their own names within RestoreBound, one restore at a time. It says the outcome on // the event snapshot-restored under the request and the act. // Verbs a request names for the snapshot restore. const ( VerbSnapshotRestore = meshbroker.VerbSnapshotRestore VerbSnapshotRestoreCancel = meshbroker.VerbSnapshotRestoreCancel VerbSnapshotRemove = meshbroker.VerbSnapshotRemove ) // RestoreBound is the longest a restore runs: past it the keeper ends it as failed. const RestoreBound = 60 * time.Minute // SnapshotRestored is the body of the event snapshot-restored. type SnapshotRestored = meshbroker.SnapshotRestoredEvent // restoreRun is the restore that runs: its act and how to end it. type restoreRun struct { act string request string streams []string cancel context.CancelFunc cancelled bool } // snapshotOf reads the recorded snapshot of request id: the refusals §681 words, or the record. func (k *Keeper) snapshotOf(id string) (Request, error) { raw, _, found, err := k.Store.Get(StateRequests, id) if err != nil { return Request{}, fmt.Errorf("the snapshot of the request %s cannot be read: %w", id, err) } var rec Request if !found || json.Unmarshal(raw, &rec) != nil || rec.Verb == "" { return Request{}, fmt.Errorf("no snapshot of the request %s is held", id) } if !rec.Recorded() { return Request{}, fmt.Errorf("snapshot taken before milestone G") } return rec, nil } // checked reads the snapshot file against its record: missing, or damaged. func (k *Keeper) checked(rec Request) error { size, digest, err := k.files().Digest(rec.Path) if errors.Is(err, os.ErrNotExist) { return fmt.Errorf("snapshot missing") } if err != nil { return fmt.Errorf("the snapshot %s cannot be read: %w", rec.Path, err) } if digest != rec.Digest || size != rec.Size { return fmt.Errorf("snapshot damaged: request %s", rec.ID) } return nil } // startRestore answers a snapshot-restore at once: accepted with the act's id (the request's own id), busy with the id of the // running one, or an error; verify says the file holds and restores nothing. func (k *Keeper) startRestore(r *Request) (string, error) { rec, err := k.snapshotOf(r.Of) if err != nil { return "error", err } if err := k.checked(rec); err != nil { return "error", err } if r.Verify { return "verified", nil } ctx, cancel := context.WithTimeout(context.Background(), RestoreBound) streams, err := k.Bus.SnapshotStreams(rec.Path) if err != nil { cancel() return "error", fmt.Errorf("the snapshot's streams cannot be read: %w", err) } for _, name := range streams { live, err := k.Bus.Live(ctx, name) if err != nil { cancel() return "error", fmt.Errorf("whether %s is live cannot be read: %w", name, err) } if live { cancel() return "error", fmt.Errorf("names live: the stream %s exists, and a restore goes over nothing", name) } } k.mu.Lock() if run := k.restoring; run != nil { k.mu.Unlock() cancel() r.ActID = run.act return "busy", nil } run := &restoreRun{act: r.ID, request: rec.ID, streams: streams, cancel: cancel} k.restoring = run k.mu.Unlock() r.ActID = run.act // The act's record is written before the restore runs, as the end of the restore and a restart's cut-off read it; a record that // cannot be written ends the start, so no restore runs that nothing could find again. held := *r held.Done, held.Outcome = k.now(), "accepted" data, _ := json.Marshal(held) if _, err := k.Store.Put(StateRequests, r.ID, data); err != nil { k.mu.Lock() k.restoring = nil k.mu.Unlock() cancel() return "error", fmt.Errorf("the restore's record cannot be written, so nothing is restored: %w", err) } k.record(r, fmt.Sprintf("restores the snapshot of the request %s (%s) in place", rec.ID, rec.Path)) go k.restore(ctx, run, rec, r.Caller, r.Why) return "accepted", nil } // restore runs the restore to its end. A restore that failed, passed its bound or was cancelled removes the keeper's own // partial stream (it held no live data, names live having come first) and says snapshot-restored failed before the keeper // takes another restore. func (k *Keeper) restore(ctx context.Context, run *restoreRun, rec Request, caller, why string) { err := k.Bus.Restore(ctx, rec.Path) k.mu.Lock() cancelled := run.cancelled k.mu.Unlock() switch { case cancelled: err = fmt.Errorf("cancelled") case errors.Is(ctx.Err(), context.DeadlineExceeded): err = fmt.Errorf("not finished within %s: %v", RestoreBound, err) } run.cancel() k.endRestore(run.request, run.act, run.streams, err) k.mu.Lock() if k.restoring == run { k.restoring = nil } k.mu.Unlock() } // endRestore ends the act: a failure removes the partial streams first, then the event says it and the request's record // holds how it ended. func (k *Keeper) endRestore(request, act string, streams []string, failure error) { result, text := meshbroker.SnapshotRestoredEventOutcomeDone, "" if failure != nil { result, text = meshbroker.SnapshotRestoredEventOutcomeFailed, failure.Error() // Removing the partial stream does not depend on the restore's own context, which has ended. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) for _, name := range streams { if err := k.Bus.DeleteStream(ctx, name); err != nil { text += fmt.Sprintf("; the partial stream %s could not be removed: %v", name, err) k.logf("the partial stream %s of the restore %s cannot be removed: %v", name, act, err) } } cancel() } k.logf("the restore %s of the snapshot of the request %s ended %s %s", act, request, result, text) if raw, _, found, err := k.Store.Get(StateRequests, act); err == nil && found { var r Request if json.Unmarshal(raw, &r) == nil { r.Ended, r.Result = k.now(), result if text != "" { r.Error = text } data, _ := json.Marshal(r) if _, err := k.Store.Put(StateRequests, act, data); err != nil { k.logf("the end of the restore %s cannot be written: %v", act, err) } } } if k.Emit == nil { return } s := Saying{Event: meshbroker.EventSnapshotRestored, ID: EventID(), At: k.now(), Restored: &SnapshotRestored{Request: request, Act: act, Outcome: result, Error: text}} if err := k.Emit(&s); err != nil { s.Error = err.Error() k.setStatus(func(st *Status) { st.Unsaid = append(st.Unsaid, s) }) k.logf("the event %s of the restore %s could not be said, said again at the next fill pass: %v", s.Event, act, err) } } // cancelRestore cancels the restore of the act r.Of names: accepted with its id, or an error when it has ended. func (k *Keeper) cancelRestore(r *Request) (string, error) { k.mu.Lock() run := k.restoring if run == nil || run.act != r.Of { k.mu.Unlock() return "error", fmt.Errorf("the restore %s has ended or is none of this keeper's running", r.Of) } run.cancelled = true k.mu.Unlock() run.cancel() r.ActID = run.act k.record(r, fmt.Sprintf("cancelled the restore %s", run.act)) return "accepted", nil } // EndCutOffRestores ends, at the keeper's start, every restore a restart cut off: its partial stream removed and // snapshot-restored said failed under its act. func (k *Keeper) EndCutOffRestores() { keys, err := k.Store.Keys(StateRequests) if err != nil { k.logf("the restores a restart cut off cannot be looked for: %v", err) return } for _, key := range keys { raw, _, found, err := k.Store.Get(StateRequests, key) if err != nil || !found { continue } var r Request if json.Unmarshal(raw, &r) != nil || r.Verb != VerbSnapshotRestore || r.Outcome != "accepted" || !r.Ended.IsZero() { continue } var streams []string if rec, err := k.snapshotOf(r.Of); err == nil { streams, _ = k.Bus.SnapshotStreams(rec.Path) } k.endRestore(r.Of, r.ID, streams, fmt.Errorf("cut off by the keeper's restart")) } }