package keeper import ( "context" "encoding/json" "errors" "fmt" "strings" "time" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" ) // A person's request (design 25 § 12.3; CAT-N8): the bus module's tools hold no grant of the seat's acting verbs, so // each hands its request to the keeper through the keeper's own state requests, stamped with the caller the tool runner // proved (never a name the request's body gives), and reads the keeper's answer back from the same key. create and // apply act at once; recreate and drop only once the operator's warrant on the ask the tool raised is held, after a // snapshot of the entry's streams. Every act is recorded as a hand act. // The verbs a request names. const ( VerbCreate = "create" VerbApply = "apply" VerbRecreate = "recreate" VerbDrop = "drop" // VerbFirstViewAgain asks the indexer's first view at once, by hand (design 25 § 12.2), beside the backoff. VerbFirstViewAgain = "first-view-again" ) // Request is one person's request, the value of its key in StateRequests. type Request struct { ID string `json:"id"` Verb string `json:"verb"` Module string `json:"module"` Local string `json:"local"` Why string `json:"why"` // Caller is who asked, as the tool runner proved it (stdio.CallerOf), written by the tool. Caller string `json:"caller"` At time.Time `json:"at"` // Ask is the id of the operator's ask the tool raised for a recreate or a drop; the keeper acts only on its warrant. Ask string `json:"ask,omitempty"` // Done is when the keeper ended it, Outcome what it did, Error why not. Done time.Time `json:"done,omitempty"` Outcome string `json:"outcome,omitempty"` Error string `json:"error,omitempty"` Snapshot string `json:"snapshot,omitempty"` // Waiting is why it waits, while it waits: the operator's warrant, or the keeper's first view. Waiting string `json:"waiting,omitempty"` // Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged. Unrecorded string `json:"unrecorded,omitempty"` } // AskLasts is how long the operator's ask for a recreate or a drop waits for an answer: shorter than the day the state // requests keeps a key, so a request whose ask ended unanswered is ended by the keeper, saying so, before its key ages // out, and the warrant's acted key (ActedKey) outlives the ask it spends. const AskLasts = 12 * time.Hour // ActedPrefix started the spend record's key while it was kept in StateRequests (`acted.`); a key of that form // still met there is no request and is skipped. const ActedPrefix = "acted." // StateSpent is the keeper process's own state of the warrants it spent, one key per ask id, claimed by compare-and-set // before a recreate or a drop deletes anything (design 25 § 12.3: a warrant acts once). Only the process writes it: the // tools bundle, which writes StateRequests as the tool runner's account, may read it and never delete a spend record // (review round 2, security F2). const StateSpent = "spent" // Acted is the value of an acted key: which request spent the warrant, on what. type Acted struct { Ask string `json:"ask"` Request string `json:"request"` Verb string `json:"verb"` Module string `json:"module"` Local string `json:"local"` At time.Time `json:"at"` } // Act is what an option of the operator's ask binds for a recreate or a drop: the verb and the entry (occ.ActDigest). func (r Request) Act() occ.Act { return occ.Act{"verb": r.Verb, "module": r.Module, "local": r.Local} } // ErrNoWarrant is a request that waits for the operator's word: kept, acted on once the warrant is held. var ErrNoWarrant = errors.New("the operator has not answered yet") // ActOnRequests acts on every request not ended yet, oldest first, and writes each one's end back to its key. func (k *Keeper) ActOnRequests() { keys, err := k.Store.Keys(StateRequests) if err != nil { k.logf("the persons' requests cannot be read: %v", err) return } for _, key := range keys { if strings.HasPrefix(key, ActedPrefix) { continue } raw, _, found, err := k.Store.Get(StateRequests, key) if err != nil || !found { continue } var r Request if json.Unmarshal(raw, &r) != nil || !r.Done.IsZero() { continue } outcome, err := k.act(&r) if errors.Is(err, ErrNoWarrant) { // Said on the request while it waits, once (its write wakes the watch, which then finds the same words). if r.Waiting != err.Error() { r.Waiting = err.Error() data, _ := json.Marshal(r) if _, err := k.Store.Put(StateRequests, key, data); err != nil { k.logf("why the request %s waits cannot be written: %v", key, err) } } continue } r.Waiting = "" r.Done, r.Outcome = k.now(), outcome if err != nil { r.Error = err.Error() } data, _ := json.Marshal(r) if _, err := k.Store.Put(StateRequests, key, data); err != nil { k.logf("the end of the request %s cannot be written: %v", key, err) } } } func (k *Keeper) act(r *Request) (string, error) { if r.Verb == VerbFirstViewAgain { _, err := k.AskFirstView(0) s := k.Status().FirstView k.record(r, fmt.Sprintf("asked %s.%s again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision)) if err != nil { return "asked, not answered", err } return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil } if !k.Viewed() { return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant) } held, _, found := k.asked(r.Module) if !found { return "nothing done", fmt.Errorf("the bus module holds no entry of %s", r.Module) } m, made := held.Made[r.Local] asked := false for _, e := range held.Entries { asked = asked || e.Name == r.Local } switch r.Verb { case VerbCreate, VerbApply: if !asked { return "nothing done", fmt.Errorf("no accepted manifest of %s asks for %s: the bus module creates only what one asks", r.Module, r.Local) } for _, e := range held.Entries { if e.Name == r.Local { k.makeEntry(&held, e) } } k.write(held) m = held.Made[r.Local] k.record(r, fmt.Sprintf("%s of %s's %s: created %v, drift %q, waiting %q", r.Verb, r.Module, r.Local, m.Created, m.Drift, m.Waiting)) if m.Refused != "" || m.Waiting != "" { return "not made", fmt.Errorf("%s%s", m.Refused, m.Waiting) } return fmt.Sprintf("%s: the bus holds %s as asked%s", r.Verb, BusName(r.Module, r.Local), map[bool]string{true: ", but for a drift a person decides: " + m.Drift}[m.Drift != ""]), nil case VerbRecreate, VerbDrop: default: return "nothing done", fmt.Errorf("%q is no verb a person asks the bus module for: create, apply, recreate or drop", r.Verb) } if !made || !m.Created { return "nothing done", fmt.Errorf("the bus module made no %s of %s", r.Local, r.Module) } if r.Verb == VerbDrop && !m.Orphaned { return "nothing done", fmt.Errorf("%s of %s is asked for by its manifest: drop removes an orphan alone", r.Local, r.Module) } if r.Verb == VerbRecreate && m.Orphaned { return "nothing done", fmt.Errorf("%s of %s is an orphan: drop it, or ask for it in a manifest again", r.Local, r.Module) } if err := k.warranted(r); err != nil { return "nothing done", err } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() var streams []string for _, o := range m.Plan.Objects { streams = append(streams, o.Stream()) } path, err := k.Bus.Snapshot(ctx, streams) if err != nil { return "nothing done", fmt.Errorf("the snapshot before %s failed, so nothing was removed: %w", r.Verb, err) } r.Snapshot = path // The warrant is spent before anything is deleted: of two requests naming one ask, only the one whose claim stands acts. if err := k.spend(r); err != nil { return "nothing done", err } for _, o := range m.Plan.Objects { if err := k.Bus.Delete(ctx, o); err != nil { return "partly done", fmt.Errorf("%s could not be removed after its snapshot at %s: %w", o.Stream(), path, err) } } if r.Verb == VerbDrop { delete(held.Made, r.Local) k.write(held) k.emit(EventDropped, m.Plan, held.Revision, r.Why, nil, nil) k.clear(conditionOrphaned, r.Module, r.Local) k.record(r, fmt.Sprintf("dropped %s of %s on the operator's warrant (snapshot %s)", r.Local, r.Module, path)) return fmt.Sprintf("dropped %s, its snapshot at %s", BusName(r.Module, r.Local), path), nil } for _, o := range m.Plan.Objects { if err := k.Bus.Create(ctx, o); err != nil { m.Created, m.Refused = false, fmt.Sprintf("%s: %v", o.Stream(), err) held.Made[r.Local] = m k.write(held) return "partly done", fmt.Errorf("%s was removed and could not be made again (its snapshot at %s): %w", o.Stream(), path, err) } } wasDrift := m.Drift != "" m.Drift = "" held.Made[r.Local] = m k.write(held) k.emit(EventRecreated, m.Plan, held.Revision, r.Why, nil, nil) if wasDrift { k.clear(conditionDrifted, r.Module, r.Local) } k.record(r, fmt.Sprintf("recreated %s of %s on the operator's warrant (snapshot %s)", r.Local, r.Module, path)) return fmt.Sprintf("recreated %s as asked, its old data's snapshot at %s", BusName(r.Module, r.Local), path), nil } // warranted holds a recreate or a drop to the operator's warrant on the ask its tool raised: chosen by the operator, // on that ask, binding exactly this act. func (k *Keeper) warranted(r *Request) error { if r.Ask == "" || k.Warrant == nil { return fmt.Errorf("a %s acts only on the operator's warrant, and this request names no ask", r.Verb) } ask, w, err := k.Warrant(r.Ask) if err != nil { return fmt.Errorf("%w: the router's record of the ask %s cannot be read now: %v", ErrNoWarrant, r.Ask, err) } // An ask that ended unanswered ends its request, said: never a request that waits until its key ages out. And a // warrant is not acted on once its ask ended, so a request handed after its acted key aged out acts on nothing. var expires time.Time if !r.At.IsZero() { expires = r.At.Add(AskLasts) } if ask != nil { if t, err := occ.Time(ask.Expires); err == nil && !t.IsZero() { expires = t } } if !expires.IsZero() && k.now().After(expires) { return fmt.Errorf("the operator's ask %s ended at %s with no warrant acted on, so nothing is removed: ask again with a new request", r.Ask, expires.UTC().Format(time.RFC3339)) } if ask == nil || w == nil { return ErrNoWarrant } o, err := occ.For(*w, Module, *ask) if err != nil { return fmt.Errorf("no warrant: %w", err) } if o.Level != occ.Destroy { return fmt.Errorf("the operator chose %s, so nothing is removed", o.Label) } return occ.Performs(o, r.Act()) } // spend claims the warrant of r's ask by compare-and-set; a warrant already spent is refused, naming what it did. func (k *Keeper) spend(r *Request) error { raw, _ := json.Marshal(Acted{Ask: r.Ask, Request: r.ID, Verb: r.Verb, Module: r.Module, Local: r.Local, At: k.now()}) _, err := k.Store.Create(StateSpent, r.Ask, raw) if err == nil { return nil } if !errors.Is(err, ErrExists) { return fmt.Errorf("the warrant of the ask %s cannot be spent now, so nothing was removed: %w", r.Ask, err) } var a Acted if got, _, found, _ := k.Store.Get(StateSpent, r.Ask); found { _ = json.Unmarshal(got, &a) } return fmt.Errorf("the operator's warrant on the ask %s already acted (%s of %s's %s, request %s, at %s): a warrant acts once, so nothing was removed", r.Ask, a.Verb, a.Module, a.Local, a.Request, a.At.Format(time.RFC3339)) } // record records what a request did as a hand act; one that cannot be recorded is said on the request's end. func (k *Keeper) record(r *Request, what string) { if k.Act == nil { return } if err := k.Act(what+", asked by "+r.Caller, r.Why, "nats_"+strings.ReplaceAll(r.Verb, "-", "_")); err != nil { r.Unrecorded = err.Error() k.logf("the hand act %q could not be recorded: %v", what, err) } }