mesh-catalog/modules/nats/internal/keeper/restore.go

package keeper

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"time"

	"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
)

// The keeper's restore in place of a snapshot it took (design 47 G §681, ADR 0675; the bus module hands the work through
// the state requests, A-45): the recovery module asks the seat mesh-broker's snapshot-restore for a protected snapshot, which
// is never handed out. The keeper checks the file against its recorded digest, refuses while a live stream of the snapshot's
// names exists, and restores them under their own names within RestoreBound, one restore at a time. It says the outcome on
// the event snapshot-restored under the request and the act.

// Verbs a request names for the snapshot restore.
const (
	VerbSnapshotRestore       = meshbroker.VerbSnapshotRestore
	VerbSnapshotRestoreCancel = meshbroker.VerbSnapshotRestoreCancel
	VerbSnapshotRemove        = meshbroker.VerbSnapshotRemove
)

// RestoreBound is the longest a restore runs: past it the keeper ends it as failed.
const RestoreBound = 60 * time.Minute

// SnapshotRestored is the body of the event snapshot-restored.
type SnapshotRestored = meshbroker.SnapshotRestoredEvent

// restoreRun is the restore that runs: its act and how to end it.
type restoreRun struct {
	act       string
	request   string
	streams   []string
	cancel    context.CancelFunc
	cancelled bool
}

// snapshotOf reads the recorded snapshot of request id: the refusals §681 words, or the record.
func (k *Keeper) snapshotOf(id string) (Request, error) {
	raw, _, found, err := k.Store.Get(StateRequests, id)
	if err != nil {
		return Request{}, fmt.Errorf("the snapshot of the request %s cannot be read: %w", id, err)
	}
	var rec Request
	if !found || json.Unmarshal(raw, &rec) != nil || rec.Verb == "" {
		return Request{}, fmt.Errorf("no snapshot of the request %s is held", id)
	}
	if !rec.Recorded() {
		return Request{}, fmt.Errorf("snapshot taken before milestone G")
	}
	return rec, nil
}

// checked reads the snapshot file against its record: missing, or damaged.
func (k *Keeper) checked(rec Request) error {
	size, digest, err := k.files().Digest(rec.Path)
	if errors.Is(err, os.ErrNotExist) {
		return fmt.Errorf("snapshot missing")
	}
	if err != nil {
		return fmt.Errorf("the snapshot %s cannot be read: %w", rec.Path, err)
	}
	if digest != rec.Digest || size != rec.Size {
		return fmt.Errorf("snapshot damaged: request %s", rec.ID)
	}
	return nil
}

// startRestore answers a snapshot-restore at once: accepted with the act's id (the request's own id), busy with the id of the
// running one, or an error; verify says the file holds and restores nothing.
func (k *Keeper) startRestore(r *Request) (string, error) {
	rec, err := k.snapshotOf(r.Of)
	if err != nil {
		return "error", err
	}
	if err := k.checked(rec); err != nil {
		return "error", err
	}
	if r.Verify {
		return "verified", nil
	}
	ctx, cancel := context.WithTimeout(context.Background(), RestoreBound)
	streams, err := k.Bus.SnapshotStreams(rec.Path)
	if err != nil {
		cancel()
		return "error", fmt.Errorf("the snapshot's streams cannot be read: %w", err)
	}
	for _, name := range streams {
		live, err := k.Bus.Live(ctx, name)
		if err != nil {
			cancel()
			return "error", fmt.Errorf("whether %s is live cannot be read: %w", name, err)
		}
		if live {
			cancel()
			return "error", fmt.Errorf("names live: the stream %s exists, and a restore goes over nothing", name)
		}
	}
	k.mu.Lock()
	if run := k.restoring; run != nil {
		k.mu.Unlock()
		cancel()
		r.ActID = run.act
		return "busy", nil
	}
	run := &restoreRun{act: r.ID, request: rec.ID, streams: streams, cancel: cancel}
	k.restoring = run
	k.mu.Unlock()
	r.ActID = run.act
	// The act's record is written before the restore runs, as the end of the restore and a restart's cut-off read it; a record that
	// cannot be written ends the start, so no restore runs that nothing could find again.
	held := *r
	held.Done, held.Outcome = k.now(), "accepted"
	data, _ := json.Marshal(held)
	if _, err := k.Store.Put(StateRequests, r.ID, data); err != nil {
		k.mu.Lock()
		k.restoring = nil
		k.mu.Unlock()
		cancel()
		return "error", fmt.Errorf("the restore's record cannot be written, so nothing is restored: %w", err)
	}
	k.record(r, fmt.Sprintf("restores the snapshot of the request %s (%s) in place", rec.ID, rec.Path))
	go k.restore(ctx, run, rec, r.Caller, r.Why)
	return "accepted", nil
}

// restore runs the restore to its end. A restore that failed, passed its bound or was cancelled removes the keeper's own
// partial stream (it held no live data, names live having come first) and says snapshot-restored failed before the keeper
// takes another restore.
func (k *Keeper) restore(ctx context.Context, run *restoreRun, rec Request, caller, why string) {
	err := k.Bus.Restore(ctx, rec.Path)
	k.mu.Lock()
	cancelled := run.cancelled
	k.mu.Unlock()
	switch {
	case cancelled:
		err = fmt.Errorf("cancelled")
	case errors.Is(ctx.Err(), context.DeadlineExceeded):
		err = fmt.Errorf("not finished within %s: %v", RestoreBound, err)
	}
	run.cancel()
	k.endRestore(run.request, run.act, run.streams, err)
	k.mu.Lock()
	if k.restoring == run {
		k.restoring = nil
	}
	k.mu.Unlock()
}

// endRestore ends the act: a failure removes the partial streams first, then the event says it and the request's record
// holds how it ended.
func (k *Keeper) endRestore(request, act string, streams []string, failure error) {
	result, text := meshbroker.SnapshotRestoredEventOutcomeDone, ""
	if failure != nil {
		result, text = meshbroker.SnapshotRestoredEventOutcomeFailed, failure.Error()
		// Removing the partial stream does not depend on the restore's own context, which has ended.
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
		for _, name := range streams {
			if err := k.Bus.DeleteStream(ctx, name); err != nil {
				text += fmt.Sprintf("; the partial stream %s could not be removed: %v", name, err)
				k.logf("the partial stream %s of the restore %s cannot be removed: %v", name, act, err)
			}
		}
		cancel()
	}
	k.logf("the restore %s of the snapshot of the request %s ended %s %s", act, request, result, text)
	if raw, _, found, err := k.Store.Get(StateRequests, act); err == nil && found {
		var r Request
		if json.Unmarshal(raw, &r) == nil {
			r.Ended, r.Result = k.now(), result
			if text != "" {
				r.Error = text
			}
			data, _ := json.Marshal(r)
			if _, err := k.Store.Put(StateRequests, act, data); err != nil {
				k.logf("the end of the restore %s cannot be written: %v", act, err)
			}
		}
	}
	if k.Emit == nil {
		return
	}
	s := Saying{Event: meshbroker.EventSnapshotRestored, ID: EventID(), At: k.now(),
		Restored: &SnapshotRestored{Request: request, Act: act, Outcome: result, Error: text}}
	if err := k.Emit(&s); err != nil {
		s.Error = err.Error()
		k.setStatus(func(st *Status) { st.Unsaid = append(st.Unsaid, s) })
		k.logf("the event %s of the restore %s could not be said, said again at the next fill pass: %v", s.Event, act, err)
	}
}

// cancelRestore cancels the restore of the act r.Of names: accepted with its id, or an error when it has ended.
func (k *Keeper) cancelRestore(r *Request) (string, error) {
	k.mu.Lock()
	run := k.restoring
	if run == nil || run.act != r.Of {
		k.mu.Unlock()
		return "error", fmt.Errorf("the restore %s has ended or is none of this keeper's running", r.Of)
	}
	run.cancelled = true
	k.mu.Unlock()
	run.cancel()
	r.ActID = run.act
	k.record(r, fmt.Sprintf("cancelled the restore %s", run.act))
	return "accepted", nil
}

// EndCutOffRestores ends, at the keeper's start, every restore a restart cut off: its partial stream removed and
// snapshot-restored said failed under its act.
func (k *Keeper) EndCutOffRestores() {
	keys, err := k.Store.Keys(StateRequests)
	if err != nil {
		k.logf("the restores a restart cut off cannot be looked for: %v", err)
		return
	}
	for _, key := range keys {
		raw, _, found, err := k.Store.Get(StateRequests, key)
		if err != nil || !found {
			continue
		}
		var r Request
		if json.Unmarshal(raw, &r) != nil || r.Verb != VerbSnapshotRestore || r.Outcome != "accepted" || !r.Ended.IsZero() {
			continue
		}
		var streams []string
		if rec, err := k.snapshotOf(r.Of); err == nil {
			streams, _ = k.Bus.SnapshotStreams(rec.Path)
		}
		k.endRestore(r.Of, r.ID, streams, fmt.Errorf("cut off by the keeper's restart"))
	}
}