Files
mesh-delivery/modules/mesh-grants/internal/grants/takeover.go
T
jschoubben a092ee9ff9 mesh-grants writes the user list sealed to the bus node and names it in user-list.sealed; the switch turns that key; take-over state says switched only once the bus node loaded it (ADR 0547 clauses 7, 8; issue 1945)
The bus node can only take over a list that exists: each pass composes the accounts block with the vault's hashes, seals
it to the key the bus node says in its have, puts it in the part's own object bucket by digest and writes the key with
digest, revision, user count, users left out and which list to follow. take-over is refused, naming why, while no list is
sealed to the bus node's current key or the loaded list does not grant the bus node its read.

Review of fix/1945-takeover-blockers: no responders on the router's record is an error, not 'no record'; an answer whose
record cannot be reached is not closed; an ask that failed to publish is taken back at the router; an error acting on an
answer is logged. Pins mesh-sdk 175a5567.
2026-10-26 11:59:41 +01:00

492 lines
19 KiB
Go

package grants
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The switch between the controller's user list and the grants part's (ADR 0547 clause 8). The operator's word is a
// warrant: the operator's answer to an ask on a channel that proves who answered, never a caller's name, since an
// operator's session and every agent's call as the same machine account (the operator, 2026-10-26).
// Shadow is the switch's state until take-over: the bus node follows the controller's list. TookOver is its state after.
const (
Shadow = "shadow"
TookOver = "took-over"
)
// KeyRequest is the key in StateTakeover where a tool leaves its request for the process to ask.
const KeyRequest = "request"
// The two acts the operator may warrant.
const (
ActTakeOver = "take-over"
ActHandBack = "hand-back"
)
// Switch is the value of `takeover`: which list the bus node follows, every act on the switch and the ask open now.
type Switch struct {
V int `json:"v"`
State string `json:"state"`
// Open is the id of the ask the operator has not answered yet, empty when none.
Open string `json:"open,omitempty"`
// Refusing are the acts whose refusal condition was said and not cleared yet.
Refusing []string `json:"refusing,omitempty"`
Acts []Act `json:"acts"`
}
// Act is one take-over or hand-back asked, answered, done or refused.
type Act struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
// Ask is the router's ask of the act; Said what became of it: asked, switched, declined, expired, refused.
Ask string `json:"ask,omitempty"`
Said string `json:"said,omitempty"`
Refused string `json:"refused,omitempty"`
}
// Request is what a tool leaves for the process to ask: written once per act, marked raised when asked.
type Request struct {
ID string `json:"id"`
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
Raised bool `json:"raised,omitempty"`
// RaisedAt is when it was asked, and Failed why asking it failed (the ask is then closed as refused).
RaisedAt time.Time `json:"raised_at,omitempty"`
Failed string `json:"failed,omitempty"`
}
// Asker is the client of the router the part asks through (mesh-sdk asker.Client).
type Asker interface {
Ask(a asks.Ask) error
// Where is the router's record of an ask: its state and its warrant, if any.
Where(id string) (string, *asks.Warrant, error)
// Take hands a warrant to act once (a warrant the process missed is caught up through it).
Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error)
// Cancel takes an ask back at the router, so a copy the operator may still see ends there too.
Cancel(id string) error
}
// askMargin is how long past an ask's expiry the router is given to say it ended, before the part closes it itself;
// recordGrace how long a just-asked ask may be missing from the router's record.
const (
askMargin = 5 * time.Minute
recordGrace = time.Minute
)
// keptActs is how many acts the switch keeps, newest last.
const keptActs = 50
func (p *Part) readSwitch() (Switch, error) {
s := Switch{V: 1, State: Shadow}
raw, _, found, err := p.Store.Get(StateTakeover, KeySwitch)
if err != nil {
return s, err
}
if found {
_ = json.Unmarshal(raw, &s)
}
return s, nil
}
func (p *Part) writeSwitch(s Switch) error {
if len(s.Acts) > keptActs {
s.Acts = s.Acts[len(s.Acts)-keptActs:]
}
if s.Acts == nil {
s.Acts = []Act{}
}
data, _ := json.Marshal(s)
_, err := p.Store.Put(StateTakeover, KeySwitch, data)
return err
}
// SwitchState is which list the bus node follows now.
func (p *Part) SwitchState() string {
if s, err := p.readSwitch(); err == nil && s.State != "" {
return s.State
}
return Shadow
}
// TakeOver is the tool: it asks the operator to switch the bus node to the grants part's list. It writes the request,
// and the part's process asks it on the operator's channel; only the operator's yes (a warrant) runs the switch, after
// the refusals of Decide. Asking again while an ask is open is refused.
func (p *Part) TakeOver(caller, why string) (any, error) { return p.request(ActTakeOver, caller, why) }
// HandBack is the tool: it asks the operator to switch the bus node back to the controller's list.
func (p *Part) HandBack(caller, why string) (any, error) { return p.request(ActHandBack, caller, why) }
// nonce is four hex digits that keep two asks of one second apart.
func (p *Part) nonce() string {
if p.Nonce != nil {
return p.Nonce()
}
var b [2]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
func (p *Part) request(act, caller, why string) (any, error) {
s, err := p.readSwitch()
if err != nil {
return nil, fmt.Errorf("the switch cannot be read: %w", err)
}
a := Act{Act: act, By: caller, Why: why, At: p.now()}
if s.Open != "" {
return nil, fmt.Errorf("%s is already asked: the ask %s waits for the operator's answer", act, s.Open)
}
if act == ActHandBack && s.State == Shadow {
a.Refused = "nothing to hand back: the grants part is in shadow and never took over, so the bus node follows the controller's list already"
s.Acts = append(s.Acts, a)
_ = p.writeSwitch(s)
return nil, fmt.Errorf("%s", a.Refused)
}
if act == ActTakeOver && s.State == TookOver {
return nil, fmt.Errorf("the bus node follows the grants part's list already")
}
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
a.Said = "asked"
req, _ := json.Marshal(Request{ID: a.Ask, Act: act, By: caller, Why: why, At: a.At})
if _, err := p.Store.Put(StateTakeover, KeyRequest, req); err != nil {
return nil, fmt.Errorf("the request could not be written: %w", err)
}
s.Open = a.Ask
s.Acts = append(s.Acts, a)
if err := p.writeSwitch(s); err != nil {
return nil, fmt.Errorf("the act could not be recorded: %w", err)
}
// The process asks at its next pass: wake it.
_, _ = p.AskCompose(caller)
return map[string]any{"asked": a.Ask, "says": "the part's process asks the operator on their channel; only the operator's yes records the switch, and it takes effect when the bus node loads the grants part's list. grants_take_over_state shows where it stands"}, nil
}
// RaiseRequested is the process's: it asks the operator the request a tool left, once. Where the part has no asker (the
// tools) it does nothing.
func (p *Part) RaiseRequested() error {
if p.Asker == nil {
return nil
}
raw, rev, found, err := p.Store.Get(StateTakeover, KeyRequest)
if err != nil || !found {
if err != nil {
return fmt.Errorf("its own %s %s cannot be read: %v", StateTakeover, KeyRequest, err)
}
return nil
}
var r Request
if json.Unmarshal(raw, &r) != nil || r.Raised || r.Failed != "" || r.ID == "" {
return nil
}
if s, err := p.readSwitch(); err != nil || s.Open != r.ID {
return nil // an ask no longer open is not asked
}
ask, err := p.AskOf(r)
if err != nil {
return err
}
if err := p.Asker.Ask(ask); err != nil {
// Closed as refused, and the half-made record the asker kept before publishing is taken away, so nothing stays open.
r.Failed = err.Error()
data, _ := json.Marshal(r)
_, _ = p.Store.Put(StateTakeover, KeyRequest, data)
if p.Forget != nil {
_ = p.Forget(r.ID)
}
// A publish that timed out may still have reached the router: taken back there, so a yes to it cannot land on an
// ask the part has forgotten.
if cerr := p.Asker.Cancel(r.ID); cerr != nil && p.Logf != nil {
p.Logf("the ask %s that could not be published was not taken back at the router: %v", r.ID, cerr)
}
if s, rerr := p.readSwitch(); rerr == nil {
_ = p.finish(s, Act{Act: r.Act, By: "operator", At: p.now(), Ask: r.ID, Said: "refused", Refused: "the operator could not be asked: " + err.Error()})
}
return fmt.Errorf("the operator could not be asked the %s: %v", r.Act, err)
}
r.Raised, r.RaisedAt = true, p.now()
data, _ := json.Marshal(r)
if _, err := p.Store.Put(StateTakeover, KeyRequest, data); err != nil {
return fmt.Errorf("the ask %s was asked and its request could not be marked: %v", r.ID, err)
}
_ = rev
return nil
}
// Reconcile closes an ask that can stay open for ever, once per pass while one is open: a warrant the process missed is
// caught up from the router's record, one that ended with none is closed, and one past its expiry and a margin is closed
// as expired by the part itself.
func (p *Part) Reconcile() error {
if p.Asker == nil {
return nil
}
s, err := p.readSwitch()
if err != nil || s.Open == "" {
return err
}
act := actOf(s.Open)
at := p.now()
for _, a := range s.Acts {
if a.Ask == s.Open {
at = a.At
break
}
}
state, w, err := p.Asker.Where(s.Open)
if err != nil {
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", s.Open, err)
}
if w != nil {
h, herr := p.Asker.Take(*w, p.Warranted)
if h == "" && herr != nil {
// The part's own record of the ask could not be reached: nothing was taken, and the next pass tries again.
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", s.Open, herr)
}
p.Said(h, *w, herr)
if cur, rerr := p.readSwitch(); rerr == nil && cur.Open == s.Open {
_ = p.finish(cur, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(w.Outcome)})
}
return nil
}
raw, _, found, _ := p.Store.Get(StateTakeover, KeyRequest)
var r Request
if found {
_ = json.Unmarshal(raw, &r)
}
switch {
case p.now().After(at.Add(asks.ApproveLasts + askMargin)):
return p.finish(s, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: string(asks.OutcomeExpired)})
case state == "" && r.ID == s.Open && r.Raised && p.now().After(r.RaisedAt.Add(recordGrace)):
return p.finish(s, Act{Act: act, By: "operator", At: p.now(), Ask: s.Open, Said: "ended", Refused: "the router holds no record of the ask"})
}
return nil
}
func actOf(id string) string {
if strings.HasPrefix(id, ActHandBack+"-") {
return ActHandBack
}
return ActTakeOver
}
// Following says whether the bus node has loaded the grants part's list: "yes" when its loaded digest equals the part's,
// "not yet" when it loaded another, "unknown" when its have says none.
func (p *Part) Following() string {
loaded := ReadInputs(p.Read, &p.cache).Loaded
g, _, had := p.Sealed()
if loaded == nil || !had || g.Digest == "" {
return "unknown: the bus module's have names no loaded list, or the grants part wrote none"
}
if loaded.Source == delivery.GrantsUserListSource && loaded.Digest == g.Digest {
return "yes"
}
return "not yet: the bus node loaded another list"
}
// AskOf is the ask of a request, in the words the operator reads: what changes, why, what it risks. Its yes binds the act.
func (p *Part) AskOf(r Request) (asks.Ask, error) {
verb, headline, does, stays, expl := "grants_take_over", "Switch the bus's user list to the grants part?", "The switch is recorded; it takes effect when the bus node loads the grants part's list.",
"The controller's list stays in force.", "The grants part has composed the user list the bus holds today. Your yes records the switch; it takes effect when the bus node loads the part's list.\n"+
"If the lists differ in any user, or a user has no hash in the vault, the switch is refused and you are told which."
if r.Act == ActHandBack {
verb, headline, does, stays, expl = "grants_hand_back", "Give the bus's user list back to the controller?", "The switch back is recorded; it takes effect when the bus node loads the controller's list.",
"The grants part's list stays in force.", "The bus follows the grants part's list now.\nThe controller still writes its list until it loses its lease, so the bus can follow it again."
}
bind, err := asks.ActDigest(asks.Act{"verb": verb, "module": "mesh-grants", "ask": r.ID})
if err != nil {
return asks.Ask{}, err
}
return asks.Ask{ID: r.ID, Headline: headline, Explanation: expl, Who: asks.Operator,
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: p.now().Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + r.Act,
Details: "Asked by " + r.By + ": " + r.Why}, nil
}
// Warranted is the process's act on the operator's answer (asker.Client.Listen hands it once, after checking the ask, the
// option and its level). A no is recorded; a yes runs the switch after the refusals, and a refusal is said to the operator.
func (p *Part) Warranted(o asks.Option, w asks.Warrant) error {
s, err := p.readSwitch()
if err != nil {
return err
}
act := ActTakeOver
if strings.HasPrefix(w.Ask, ActHandBack+"-") {
act = ActHandBack
}
if s.Open != w.Ask {
return fmt.Errorf("the warrant answers the ask %s, and the ask open is %q: nothing is switched", w.Ask, s.Open)
}
a := Act{Act: act, By: "operator", At: p.now(), Ask: w.Ask}
if o.ID != "yes" {
a.Said = "declined"
return p.finish(s, a)
}
var reasons []string
if act == ActTakeOver {
reasons = p.Decide()
if s, err = p.readSwitch(); err != nil { // Decide clears a refusal and writes the switch: read it again
return err
}
} else if s.State == Shadow {
reasons = []string{"nothing to hand back: the bus node follows the controller's list already"}
}
if len(reasons) > 0 {
a.Said, a.Refused = "refused", strings.Join(reasons, "; ")
p.sayRefused(&s, act, a.Refused)
return p.finish(s, a)
}
p.clearRefused(&s)
a.Said = "recorded"
if act == ActTakeOver {
s.State = TookOver
} else {
s.State = Shadow
}
if err := p.finish(s, a); err != nil {
return err
}
// The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8).
if err := p.syncFollow(s.State); err != nil {
return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err)
}
if p.Emit != nil {
_ = p.Emit.Emit(EventUserList, map[string]any{"switch": s.State})
}
return nil
}
// Said is the process's note of an ask that ended with no yes or no (expired, or refused by the router): the switch
// stays, and the ask is closed so the operator can be asked again.
func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) {
if err != nil && p.Logf != nil {
p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err)
}
s, err := p.readSwitch()
if err != nil || s.Open != w.Ask {
return
}
_ = p.finish(s, Act{Act: actOf(w.Ask), By: "operator", At: p.now(), Ask: w.Ask, Said: string(w.Outcome) + " (" + string(h) + ")"})
}
func (p *Part) finish(s Switch, a Act) error {
if s.Open == a.Ask {
s.Open = ""
}
s.Acts = append(s.Acts, a)
return p.writeSwitch(s)
}
// Decide is what stands between a take-over and the grants part's list, read from a composition made now: the loaded list
// not read, any user the two lists keep apart (one the loaded list has that the shadow cannot compose among them, such as
// an enrolment user or a person whose records are in the controller's store alone), and any user with no hash in the vault.
// Nothing here but what is true now: a take-over is refused for these and for no other reason. Empty means none.
func (p *Part) Decide() []string {
p.deciding.Store(true)
res := p.Compose()
p.deciding.Store(false)
var reasons []string
if c := res.UserList.Compare; !c.Equal() {
if c.Unread != "" {
reasons = append(reasons, "the loaded list could not be read: "+c.Unread)
}
if d := c.Differing(); len(d) > 0 {
reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list: %s", len(d), strings.Join(d, ", ")))
}
}
reasons = append(reasons, p.followReasons(res, ReadInputs(p.Read, &p.cache))...)
cr, gone := res.UserList.Credentials, res.UserList.Compare.Gone
var unhashed []string
for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) {
if !contains(gone, u) {
unhashed = append(unhashed, u)
}
}
if len(unhashed) > 0 {
sort.Strings(unhashed)
reasons = append(reasons, fmt.Sprintf("the vault holds no hash for %d users (ADR 0547 clause 9: handed or adopted before take-over): %s",
len(unhashed), strings.Join(unhashed, ", ")))
}
if len(reasons) == 0 {
if s, err := p.readSwitch(); err == nil && len(s.Refusing) > 0 {
p.clearRefused(&s)
_ = p.writeSwitch(s)
}
}
return reasons
}
func contains(s []string, v string) bool {
for _, x := range s {
if x == v {
return true
}
}
return false
}
// sayRefused tells the operator a yes was given and the switch refused, with the reasons (the users that differ named). The
// condition stays raised until a switch is recorded or a later Decide finds no reason (clearRefused).
func (p *Part) sayRefused(s *Switch, act, why string) {
if p.Say == nil {
return
}
c := asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"},
State: asks.StateRaised, Summary: act + " refused: " + why, Headline: "The user list switch was refused", Needs: "a look at why",
Explanation: asks.NeedsYou + "a look at why. You said yes to the switch and it was refused: " + why, Resolved: "the switch is asked again and goes through"}
if p.Say(c) == nil && !contains(s.Refusing, act) {
s.Refusing = append(s.Refusing, act)
}
}
// clearRefused clears every refusal condition said and not cleared: on a recorded switch, a hand-back, or a Decide with no reason.
func (p *Part) clearRefused(s *Switch) {
if p.Say == nil {
return
}
var left []string
for _, act := range s.Refusing {
c := asks.Condition{Key: "mesh-grants." + act + "-refused", Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}, State: asks.StateCleared}
if p.Say(c) != nil {
left = append(left, act)
}
}
s.Refusing = left
}
// TakeOverState is the tool take-over-state: the switch, the request, and what became of each ask.
func (p *Part) TakeOverState() (any, error) {
s, err := p.readSwitch()
if err != nil {
return nil, err
}
loaded := ReadInputs(p.Read, &p.cache).Loaded
out := map[string]any{"state": s.State, "open": s.Open, "acts": s.Acts, "bus_node_loaded_grants_list": p.Following(),
"effect": p.Effect(s.State, loaded),
"says": "a recorded switch takes effect when the bus node loads the grants part's list; effect says switched only once the bus node's loaded digest is the grants part's written one"}
if g, rev, had := p.Sealed(); had {
out["written"] = map[string]any{"digest": g.Digest, "revision": g.Revision, "key_revision": rev, "follow": g.Follow,
"users": g.Users, "left_out": g.LeftOut, "sealed_to": g.Node}
}
if loaded != nil {
out["loaded"] = map[string]any{"source": loaded.Source, "digest": loaded.Digest, "revision": loaded.Revision,
"users": loaded.Users, "follows": loaded.Follows, "at": loaded.At, "error": loaded.Error}
}
if raw, _, found, err := p.Store.Get(StateTakeover, KeyRequest); err == nil && found {
out["request"] = json.RawMessage(raw)
}
return out, nil
}