mesh-catalog/modules/nats/internal/keeper/requests.go

package keeper

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
	oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)

// A person's request (design 25 § 12.3; CAT-N8): the bus module's tools hold no grant of the seat's acting verbs, so
// each hands its request to the keeper through the keeper's own state requests, stamped with the caller the tool runner
// proved (never a name the request's body gives), and reads the keeper's answer back from the same key. create and
// apply act at once; recreate and drop only once the operator's warrant on the ask the tool raised is held, after a
// snapshot of the entry's streams. Every act is recorded as a hand act.

// The verbs a request names.
const (
	VerbCreate   = "create"
	VerbApply    = "apply"
	VerbRecreate = "recreate"
	VerbDrop     = "drop"
	// VerbFirstViewAgain asks the catalog half's first views again at once, by hand (design 47 G §614).
	VerbFirstViewAgain = "first-view-again"
)

// Request is one person's request, the value of its key in StateRequests.
type Request struct {
	ID     string `json:"id"`
	Verb   string `json:"verb"`
	Module string `json:"module"`
	Local  string `json:"local"`
	Why    string `json:"why"`
	// Caller is who asked, as the tool runner proved it (stdio.CallerOf), written by the tool.
	Caller string    `json:"caller"`
	At     time.Time `json:"at"`
	// Ask is the id of the operator's ask the tool raised for a recreate or a drop; the keeper acts only on its warrant.
	Ask string `json:"ask,omitempty"`
	// Done is when the keeper ended it, Outcome what it did, Error why not.
	Done     time.Time `json:"done,omitempty"`
	Outcome  string    `json:"outcome,omitempty"`
	Error    string    `json:"error,omitempty"`
	Snapshot string    `json:"snapshot,omitempty"`
	// Path, Size, Digest and Protected are the snapshot the keeper took before a drop or recreate, written before it touches
	// the live data (design 47 G §681): where it is, its bytes, the `sha256:` of them, and whether it holds a stream no other
	// account may hear, which is never handed out.
	Path      string `json:"path,omitempty"`
	Size      int64  `json:"size,omitempty"`
	Digest    string `json:"digest,omitempty"`
	Protected bool   `json:"protected,omitempty"`
	// Of is the keeper's snapshot a snapshot-restore or snapshot-remove names, by its request; Verify checks only; ActID is the
	// restore's act (the id of the running one for a busy answer); Ended is when the restore ended, and Result how: done or failed
	// (design 47 G §681).
	Of     string `json:"of,omitempty"`
	Verify bool   `json:"verify,omitempty"`
	// AllExpired is a snapshot-remove of every snapshot past its days, and Result then how many went.
	AllExpired bool      `json:"all-expired,omitempty"`
	ActID      string    `json:"act-id,omitempty"`
	Ended      time.Time `json:"ended,omitempty"`
	Result     string    `json:"result,omitempty"`
	// Waiting is why it waits, while it waits: the operator's warrant, or the keeper's first view.
	Waiting string `json:"waiting,omitempty"`
	// Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged.
	Unrecorded string `json:"unrecorded,omitempty"`
}

// AskLasts is how long the operator's ask for a recreate or a drop waits for an answer: the bound of its highest option
// level, destroy (design 47 G §570: no asker asks for longer than its level allows; the asker client refuses more). It is
// shorter than the day the state requests keeps a key, so a request whose ask ended unanswered is ended by the keeper,
// saying so, before its key ages out, and the warrant's acted key (ActedKey) outlives the ask it spends.
const AskLasts = occ.DestroyLasts

// ActedPrefix started the spend record's key while it was kept in StateRequests (`acted.<ask id>`); a key of that form
// still met there is no request and is skipped.
const ActedPrefix = "acted."

// StateSpent is the keeper process's own state of the warrants it spent, one key per ask id, claimed by compare-and-set
// before a recreate or a drop deletes anything (design 25 § 12.3: a warrant acts once). Only the process writes it: the
// tools bundle, which writes StateRequests as the tool runner's account, may read it and never delete a spend record
// (review round 2, security F2).
const StateSpent = "spent"

// Acted is the value of an acted key: which request spent the warrant, on what.
type Acted struct {
	Ask     string    `json:"ask"`
	Request string    `json:"request"`
	Verb    string    `json:"verb"`
	Module  string    `json:"module"`
	Local   string    `json:"local"`
	At      time.Time `json:"at"`
}

// Act is what an option of the operator's ask binds for a recreate or a drop: the verb and the entry (occ.ActDigest).
func (r Request) Act() occ.Act {
	return occ.Act{"verb": r.Verb, "module": r.Module, "local": r.Local}
}

// AnswerSnapshot answers one call of the seat mesh-broker's snapshot-restore, snapshot-restore-cancel or snapshot-remove at once, as
// the keeper's process answers it (design 47 G §681, A-107): r names the verb, its arguments and the proved caller, and comes back
// ended with its Outcome (accepted, busy, verified or error), ActID and Error. Nothing is handed through the state requests but the
// record of a restore that runs, which its end and a restart's cut-off read.
func (k *Keeper) AnswerSnapshot(r Request) Request {
	if r.At.IsZero() {
		r.At = k.now()
	}
	var outcome string
	var err error
	switch r.Verb {
	case VerbSnapshotRestore:
		outcome, err = k.startRestore(&r)
	case VerbSnapshotRestoreCancel:
		outcome, err = k.cancelRestore(&r)
	case VerbSnapshotRemove:
		outcome, err = k.removeSnapshots(&r)
	default:
		outcome, err = "error", fmt.Errorf("%q is no snapshot verb", r.Verb)
	}
	r.Done, r.Outcome = k.now(), outcome
	if err != nil {
		r.Error = err.Error()
	}
	return r
}

// ErrNoWarrant is a request that waits for the operator's word: kept, acted on once the warrant is held.
var ErrNoWarrant = errors.New("the operator has not answered yet")

// ActOnRequests acts on every request not ended yet, oldest first, and writes each one's end back to its key.
func (k *Keeper) ActOnRequests() {
	k.ExpireSnapshots()
	keys, err := k.Store.Keys(StateRequests)
	if err != nil {
		k.logf("the persons' requests cannot be read: %v", err)
		return
	}
	for _, key := range keys {
		if strings.HasPrefix(key, ActedPrefix) {
			continue
		}
		raw, _, found, err := k.Store.Get(StateRequests, key)
		if err != nil || !found {
			continue
		}
		var r Request
		if json.Unmarshal(raw, &r) != nil || !r.Done.IsZero() {
			continue
		}
		outcome, err := k.act(&r)
		if errors.Is(err, ErrNoWarrant) {
			// Said on the request while it waits, once (its write wakes the watch, which then finds the same words).
			if r.Waiting != err.Error() {
				r.Waiting = err.Error()
				data, _ := json.Marshal(r)
				if _, err := k.Store.Put(StateRequests, key, data); err != nil {
					k.logf("why the request %s waits cannot be written: %v", key, err)
				}
			}
			continue
		}
		r.Waiting = ""
		r.Done, r.Outcome = k.now(), outcome
		if err != nil {
			r.Error = err.Error()
		}
		data, _ := json.Marshal(r)
		if _, err := k.Store.Put(StateRequests, key, data); err != nil {
			k.logf("the end of the request %s cannot be written: %v", key, err)
		}
	}
}

func (k *Keeper) act(r *Request) (string, error) {
	if r.Verb == VerbFirstViewAgain {
		err := fmt.Errorf("the keeper follows no catalog half")
		if k.Again != nil {
			err = k.Again()
		}
		if err == nil {
			err = k.PlanCatalog()
		}
		s := k.Status().FirstView
		k.record(r, fmt.Sprintf("asked the catalog half's first views (%s.%s) again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision))
		if err != nil {
			return "asked, not answered", err
		}
		return fmt.Sprintf("the first views answered at revision %d", s.Revision), nil
	}
	if !k.Viewed() {
		return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant)
	}
	held, _, found := k.asked(r.Module)
	if !found {
		return "nothing done", fmt.Errorf("the bus module holds no entry of %s", r.Module)
	}
	m, made := held.Made[r.Local]
	asked := false
	for _, e := range held.Entries {
		asked = asked || e.Name == r.Local
	}
	switch r.Verb {
	case VerbCreate, VerbApply:
		if !asked {
			return "nothing done", fmt.Errorf("no accepted manifest of %s asks for %s: the bus module creates only what one asks", r.Module, r.Local)
		}
		for _, e := range held.Entries {
			if e.Name == r.Local {
				k.makeEntry(&held, e)
			}
		}
		k.write(held)
		m = held.Made[r.Local]
		k.record(r, fmt.Sprintf("%s of %s's %s: created %v, drift %q, waiting %q", r.Verb, r.Module, r.Local, m.Created, m.Drift, m.Waiting))
		if m.Refused != "" || m.Waiting != "" {
			return "not made", fmt.Errorf("%s%s", m.Refused, m.Waiting)
		}
		return fmt.Sprintf("%s: the bus holds %s as asked%s", r.Verb, BusName(r.Module, r.Local), map[bool]string{true: ", but for a drift a person decides: " + m.Drift}[m.Drift != ""]), nil
	case VerbRecreate, VerbDrop:
	default:
		return "nothing done", fmt.Errorf("%q is no verb a person asks the bus module for: create, apply, recreate or drop", r.Verb)
	}
	if !made || !m.Created {
		return "nothing done", fmt.Errorf("the bus module made no %s of %s", r.Local, r.Module)
	}
	if r.Verb == VerbDrop && !m.Orphaned {
		return "nothing done", fmt.Errorf("%s of %s is asked for by its manifest: drop removes an orphan alone", r.Local, r.Module)
	}
	if r.Verb == VerbRecreate && m.Orphaned {
		return "nothing done", fmt.Errorf("%s of %s is an orphan: drop it, or ask for it in a manifest again", r.Local, r.Module)
	}
	if err := k.warranted(r); err != nil {
		return "nothing done", err
	}
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()
	var streams []string
	for _, o := range m.Plan.Objects {
		streams = append(streams, o.Stream())
	}
	path, err := k.Bus.Snapshot(ctx, streams)
	if err != nil {
		return "nothing done", fmt.Errorf("the snapshot before %s failed, so nothing was removed: %w", r.Verb, err)
	}
	r.Snapshot = path
	if err := k.recordSnapshot(ctx, r, path, streams); err != nil {
		r.Snapshot = ""
		return "nothing done", err
	}
	// The warrant is spent before anything is deleted: of two requests naming one ask, only the one whose claim stands acts.
	if err := k.spend(r); err != nil {
		return "nothing done", err
	}
	for _, o := range m.Plan.Objects {
		if err := k.Bus.Delete(ctx, o); err != nil {
			return "partly done", fmt.Errorf("%s could not be removed after its snapshot at %s: %w", o.Stream(), path, err)
		}
	}
	if r.Verb == VerbDrop {
		delete(held.Made, r.Local)
		k.write(held)
		k.emit(EventDropped, m.Plan, held.Revision, r.Why, nil, nil)
		k.clear(conditionOrphaned, r.Module, r.Local)
		k.record(r, fmt.Sprintf("dropped %s of %s on the operator's warrant (snapshot %s)", r.Local, r.Module, path))
		return fmt.Sprintf("dropped %s, its snapshot at %s", BusName(r.Module, r.Local), path), nil
	}
	for _, o := range m.Plan.Objects {
		if err := k.Bus.Create(ctx, o); err != nil {
			m.Created, m.Refused = false, fmt.Sprintf("%s: %v", o.Stream(), err)
			held.Made[r.Local] = m
			k.write(held)
			return "partly done", fmt.Errorf("%s was removed and could not be made again (its snapshot at %s): %w", o.Stream(), path, err)
		}
	}
	wasDrift := m.Drift != ""
	m.Drift = ""
	held.Made[r.Local] = m
	k.write(held)
	k.emit(EventRecreated, m.Plan, held.Revision, r.Why, nil, nil)
	if wasDrift {
		k.clear(conditionDrifted, r.Module, r.Local)
	}
	k.record(r, fmt.Sprintf("recreated %s of %s on the operator's warrant (snapshot %s)", r.Local, r.Module, path))
	return fmt.Sprintf("recreated %s as asked, its old data's snapshot at %s", BusName(r.Module, r.Local), path), nil
}

// warranted holds a recreate or a drop to the operator's warrant on the ask its tool raised: chosen by the operator,
// on that ask, binding exactly this act.
func (k *Keeper) warranted(r *Request) error {
	if r.Ask == "" || k.Warrant == nil {
		return fmt.Errorf("a %s acts only on the operator's warrant, and this request names no ask", r.Verb)
	}
	ask, w, err := k.Warrant(r.Ask)
	if err != nil {
		return fmt.Errorf("%w: the router's record of the ask %s cannot be read now: %v", ErrNoWarrant, r.Ask, err)
	}
	// An ask the router refused at take-in ends its request with the router's words, never "not answered yet" (design 47 G
	// §570).
	if w != nil && w.Outcome == oc.WarrantOutcomeRefused {
		return fmt.Errorf("the router refused the operator's ask %s, so nothing is removed: %s", r.Ask, w.Words)
	}
	// An ask that ended unanswered ends its request, said: never a request that waits until its key ages out. And a
	// warrant is not acted on once its ask ended, so a request handed after its acted key aged out acts on nothing.
	var expires time.Time
	if !r.At.IsZero() {
		expires = r.At.Add(AskLasts)
	}
	if ask != nil {
		if t, err := occ.Time(ask.Expires); err == nil && !t.IsZero() {
			expires = t
		}
	}
	if !expires.IsZero() && k.now().After(expires) {
		return fmt.Errorf("the operator's ask %s ended at %s with no warrant acted on, so nothing is removed: ask again with a new request",
			r.Ask, expires.UTC().Format(time.RFC3339))
	}
	if ask == nil || w == nil {
		return ErrNoWarrant
	}
	o, err := occ.For(*w, Module, *ask)
	if err != nil {
		return fmt.Errorf("no warrant: %w", err)
	}
	if o.Level != occ.Destroy {
		return fmt.Errorf("the operator chose %s, so nothing is removed", o.Label)
	}
	return occ.Performs(o, r.Act())
}

// spend claims the warrant of r's ask by compare-and-set; a warrant already spent is refused, naming what it did.
func (k *Keeper) spend(r *Request) error {
	raw, _ := json.Marshal(Acted{Ask: r.Ask, Request: r.ID, Verb: r.Verb, Module: r.Module, Local: r.Local, At: k.now()})
	_, err := k.Store.Create(StateSpent, r.Ask, raw)
	if err == nil {
		return nil
	}
	if !errors.Is(err, ErrExists) {
		return fmt.Errorf("the warrant of the ask %s cannot be spent now, so nothing was removed: %w", r.Ask, err)
	}
	var a Acted
	if got, _, found, _ := k.Store.Get(StateSpent, r.Ask); found {
		_ = json.Unmarshal(got, &a)
	}
	return fmt.Errorf("the operator's warrant on the ask %s already acted (%s of %s's %s, request %s, at %s): a warrant acts once, so nothing was removed",
		r.Ask, a.Verb, a.Module, a.Local, a.Request, a.At.Format(time.RFC3339))
}

// record records what a request did as a hand act; one that cannot be recorded is said on the request's end.
func (k *Keeper) record(r *Request, what string) {
	if k.Act == nil {
		return
	}
	if err := k.Act(what+", asked by "+r.Caller, r.Why, "nats_"+strings.ReplaceAll(r.Verb, "-", "_")); err != nil {
		r.Unrecorded = err.Error()
		k.logf("the hand act %q could not be recorded: %v", what, err)
	}
}