package main // A node's two logins and their rotation (novox/hq ADR 0583 §1 to §7 and its insights; issue 774). // // A node-engine dials the bus as `node.` or as `node.--b`, the two logins alternating. The vault's state // `node-logins` holds one key per node, ``: the login in use, and while a rotation runs the other login with its stage // (begun, ready, confirmed). Each node-engine reads its own key alone (the bus grants it that read); the vault writes it // alone. // // `rotate-node` begins a rotation: answered only for a call that carries a hand act recorded through mesh-hand-acts, read // back by its id; one rotation at a time mesh-wide, a call asked meanwhile is queued and begun at the pass that finds the // first done; the bus node last, after every other node asked. It makes the other login's value, keeps it as // `own//mesh-host/` (sealed, with the operator copy; the node-engine takes it through `own`, sealed to its own // key), writes its hash as the key `` of bus-accounts and marks the login in use as rotating, so the grants part // composes both. `node-account-ready` is said only when the user list the bus node loaded holds the new login's hash. // `node-account-confirm` is the node-engine's, dialled as the new login, which the runtime proves: the old login's hash // is then removed, and `rotation-done` is said once the loaded list lacks it, when the old value is dropped and the // rotating part of the key cleared. `abandon` drops the new login and its value before the confirmation and keeps the // old hash. A key whose node leaves the placement is deleted. import ( "encoding/json" "errors" "fmt" "slices" "sort" "strings" "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" "golang.org/x/crypto/bcrypt" ) // The events a rotation says and the stages of a node's key are mesh-sdk's (delivery/node_logins.go); a rotation warranted // by a hand act is warranted for a day. const ( RotationBegunEvent = delivery.RotationBegunEvent NodeAccountReadyEvent = delivery.NodeAccountReadyEvent RotationDoneEvent = delivery.RotationDoneEvent StageBegun = delivery.RotationBegun StageReady = delivery.RotationReady StageConfirmed = delivery.RotationConfirmed // HandActWithin is how recent a hand act is that warrants a rotation. HandActWithin = 24 * time.Hour ) // NodeLogin is one key of the vault's state node-logins (mesh-sdk's contract): the login the node's node-engine dials as, // and while a rotation runs the other login with its stage. Never a value or a hash. type NodeLogin = delivery.NodeLogin // NodeLoginStore is the vault's state `node-logins`, by node. type NodeLoginStore interface { Get(node string) (*NodeLogin, error) Put(n NodeLogin) error Delete(node string) error Keys() ([]string, error) } // busNodeLogins is NodeLoginStore over the vault's own state through its runtime. type busNodeLogins struct{} func (busNodeLogins) Get(node string) (*NodeLogin, error) { e, err := stdio.State(NodeLoginsState).Get(node) if err != nil || e == nil { return nil, err } n, err := delivery.DecodeNodeLogin(e.Value) if err != nil { return nil, fmt.Errorf("%s %s is not a node's login: %w", NodeLoginsState, node, err) } return &n, nil } func (busNodeLogins) Put(n NodeLogin) error { raw, err := delivery.EncodeNodeLogin(n) if err != nil { return err } _, err = stdio.State(NodeLoginsState).Put(n.Node, json.RawMessage(raw)) return err } func (busNodeLogins) Delete(node string) error { return stdio.State(NodeLoginsState).Delete(node) } func (busNodeLogins) Keys() ([]string, error) { return stdio.State(NodeLoginsState).Keys() } // HandActSeen is a hand act as the vault reads it back from mesh-hand-acts by its id (mesh-sdk's handacts.Act): who // recorded it, when, and what it was (its verb and arguments). type HandActSeen struct { ID string `json:"id"` By string `json:"by"` At time.Time `json:"at"` Verb string `json:"verb"` Args []string `json:"arguments,omitempty"` } // ActReader reads a hand act back by its id: nil when mesh-hand-acts holds none. type ActReader func(id string) (*HandActSeen, error) // queuedRotation is a `rotate-node` asked while another ran, begun at the pass that finds none running. type queuedRotation struct { Node string `json:"node"` By string `json:"by"` Why string `json:"why"` Act string `json:"act"` At time.Time `json:"at"` } // nodeRotation is what the vault keeps in memory of its rotations: the user list the bus node loaded (user to hash), and // the rotations queued. The record is node-logins and bus-accounts; a restart forgets the queue, and the one who asked // asks again. type nodeRotation struct { mu sync.Mutex loaded map[string]string loadedFrom string queue []queuedRotation } func (v *Vault) getLogin(node string) (*NodeLogin, error) { if v.Logins == nil { return nil, errors.New("the vault has no state " + NodeLoginsState + " to read") } return v.Logins.Get(node) } // putLogin writes node's key; a key written without its hand acts keeps the ones recorded (ADR 0583 §2). func (v *Vault) putLogin(n NodeLogin) error { if v.Logins == nil { return errors.New("the vault has no state " + NodeLoginsState + " to write") } if n.Acts == nil { was, err := v.Logins.Get(n.Node) if err != nil { return err } if was != nil { n.Acts = was.Acts } } return v.Logins.Put(n) } func (v *Vault) deleteLogin(node string) error { if v.Logins == nil { return nil } return v.Logins.Delete(node) } // currentLogin is the login node dials as: its key's, or `node.` before any key is written. func (v *Vault) currentLogin(node string) (string, error) { nl, err := v.getLogin(node) if err != nil { return "", err } if nl != nil && nl.Login != "" { return nl.Login, nil } return NodeEnginePrefix + node, nil } // otherLogin is the login a node alternates to from login. func otherLogin(login string) string { if base, second := strings.CutSuffix(login, SecondLoginSuffix); second { return base } return login + SecondLoginSuffix } // loginNode is the node whose login a bus user is: `node.` or `node.--b`. func loginNode(user string) (string, bool) { rest, ok := strings.CutPrefix(user, NodeEnginePrefix) if !ok { return "", false } node := strings.TrimSuffix(rest, SecondLoginSuffix) return node, node != "" && enrolNode(node) == nil } // checkAct reads the hand act back by its id and refuses a rotation it does not warrant: none recorded, recorded by // another caller, older than a day, or an act that is not rotate-node of this node. func (v *Vault) checkAct(caller, node, id string) error { if strings.TrimSpace(id) == "" { return errors.New("rotate-node carries the hand act that warrants it (`act`): record it through mesh-hand-acts first (novox/hq ADR 0583 §2)") } if v.ReadAct == nil { return errors.New("the vault reads no hand acts, so it answers no rotation (novox/hq ADR 0583 §2)") } act, err := v.ReadAct(id) if err != nil { return fmt.Errorf("the hand act %s could not be read back from mesh-hand-acts: %w", id, err) } switch { case act == nil: return fmt.Errorf("mesh-hand-acts holds no hand act %s: record one first (novox/hq ADR 0583 §2)", id) case act.By != caller: return fmt.Errorf("the hand act %s was recorded by %s, not by %s who asks: a rotation is warranted by its own caller's act", id, act.By, caller) case v.Now().Sub(act.At) > HandActWithin: return fmt.Errorf("the hand act %s is older than %s: record a new one", id, HandActWithin) case act.Verb != delivery.RotateNodeVerb && !strings.HasSuffix(act.Verb, "."+delivery.RotateNodeVerb): return fmt.Errorf("the hand act %s is %q, not %s: it warrants no rotation", id, act.Verb, delivery.RotateNodeVerb) case !slices.Contains(act.Args, node): return fmt.Errorf("the hand act %s is a rotation of %s, not of %s", id, strings.Join(act.Args, " "), node) } return nil } // rotatingNodes are the nodes whose key says a rotation runs, sorted. func (v *Vault) rotatingNodes() ([]NodeLogin, error) { if v.Logins == nil { return nil, nil } keys, err := v.Logins.Keys() if err != nil { return nil, err } var out []NodeLogin for _, k := range keys { nl, err := v.Logins.Get(k) if err != nil { return nil, err } if nl != nil && nl.Rotating != "" { out = append(out, *nl) } } sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) return out, nil } // RotateNodeArgs are `rotate-node`'s arguments. type RotateNodeArgs struct { Node string Why string Act string } // RotateNodeAnswer says a rotation began, or that it waits for the ones before it. Never a value. type RotateNodeAnswer struct { Node string `json:"node"` Login string `json:"login,omitempty"` Rotating string `json:"rotating,omitempty"` Stage string `json:"stage,omitempty"` Revision uint64 `json:"list_revision,omitempty"` Queued bool `json:"queued,omitempty"` // Behind are the nodes whose rotation it waits for, running or asked before. Behind []string `json:"behind,omitempty"` } // RotateNode answers caller's `rotate-node`. func (v *Vault) RotateNode(caller string, a RotateNodeArgs) (RotateNodeAnswer, error) { if err := fromAGrantedCaller(caller, "rotate-node"); err != nil { return RotateNodeAnswer{}, err } if err := enrolNode(a.Node); err != nil { return RotateNodeAnswer{}, err } if strings.TrimSpace(a.Why) == "" { return RotateNodeAnswer{}, errors.New("rotate-node says why (`why`)") } if err := v.checkAct(caller, a.Node, a.Act); err != nil { return RotateNodeAnswer{}, err } if v.Accounts == nil { return RotateNodeAnswer{}, errors.New("the vault has no state of bus accounts to write to") } v.minting.mu.Lock() revision := v.minting.loadedRevision v.minting.mu.Unlock() if revision == 0 { return RotateNodeAnswer{}, errors.New("no rotation begins: the revision of the user list the bus loaded is not known yet") } bus, err := v.busNode() if err != nil { return RotateNodeAnswer{}, fmt.Errorf("no rotation is ordered: the node holding the bus is not known (the bus node rotates last): %w", err) } v.nr.mu.Lock() defer v.nr.mu.Unlock() if nl, err := v.getLogin(a.Node); err != nil { return RotateNodeAnswer{}, err } else if nl != nil && nl.Rotating != "" { return RotateNodeAnswer{}, fmt.Errorf("%s is rotating already, since %s (second login %s)", a.Node, nl.Since.Format(time.RFC3339), nl.Rotating) } for _, q := range v.nr.queue { if q.Node == a.Node { return RotateNodeAnswer{}, fmt.Errorf("%s's rotation is asked already, by %s, and waits its turn", a.Node, q.By) } } if err := v.recordAct(a.Node, a.Act); err != nil { return RotateNodeAnswer{}, err } running, err := v.rotatingNodes() if err != nil { return RotateNodeAnswer{}, err } var behind []string for _, r := range running { behind = append(behind, r.Node) } if a.Node == bus { for _, q := range v.nr.queue { behind = append(behind, q.Node) } } if len(behind) > 0 { v.nr.queue = append(v.nr.queue, queuedRotation{Node: a.Node, By: caller, Why: a.Why, Act: a.Act, At: v.Now().UTC()}) v.log("the rotation of %s waits for %s (asked by %s): one rotation at a time, the bus node %s last", a.Node, strings.Join(behind, ", "), caller, bus) return RotateNodeAnswer{Node: a.Node, Queued: true, Behind: behind}, nil } nl, err := v.beginNodeRotation(caller, a.Node, a.Why, a.Act, revision) if err != nil { return RotateNodeAnswer{}, err } return RotateNodeAnswer{Node: a.Node, Login: nl.Login, Rotating: nl.Rotating, Stage: nl.Stage, Revision: revision}, nil } // recordAct keeps the hand act in node's key of node-logins, refusing one already used: an act warrants one rotation // (ADR 0583 §2). Under nr.mu. func (v *Vault) recordAct(node, act string) error { if v.Logins == nil { return errors.New("the vault has no state " + NodeLoginsState + " to record the rotation in") } nl, err := v.getLogin(node) if err != nil { return err } if nl != nil && slices.Contains(nl.Acts, act) { return fmt.Errorf("the hand act %s warranted a rotation of %s already: record a new one", act, node) } if nl == nil { cur, err := v.currentLogin(node) if err != nil { return err } nl = &NodeLogin{Node: node, Login: cur, Since: v.Now().UTC()} } nl.Acts = append(slices.Clone(nl.Acts), act) if err := v.Logins.Put(*nl); err != nil { return fmt.Errorf("the hand act %s could not be recorded in %s %s: %w", act, NodeLoginsState, node, err) } return nil } // beginNodeRotation makes the other login and records the rotation; under nr.mu. func (v *Vault) beginNodeRotation(by, node, why, act string, revision uint64) (NodeLogin, error) { cur, err := v.currentLogin(node) if err != nil { return NodeLogin{}, err } curKey, err := v.Accounts.Get(cur) if err != nil { return NodeLogin{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, cur, err) } if curKey == nil { return NodeLogin{}, fmt.Errorf("%s holds no key of %s, the login %s dials as: there is nothing to rotate", BusAccountsState, cur, node) } second := otherLogin(cur) value, password, err := v.accountValue(node, second) if err != nil { return NodeLogin{}, err } secret := loginSecret(node, second) c, err := v.keep(secret, value, "the vault", "", OriginMade) if err != nil { return NodeLogin{}, fmt.Errorf("the login %s is not kept: %w", second, err) } hash, err := bcrypt.GenerateFromPassword([]byte(password), v.hashCost()) if err != nil { return NodeLogin{}, err } at := v.Now().UTC() fp := Fingerprint(value) if err := v.Accounts.Put(AccountHash{User: second, Hash: string(hash), Of: secret, Generation: c.Generation, Fingerprint: fp, At: at, Kept: KeptNode}); err != nil { return NodeLogin{}, fmt.Errorf("the hash of %s could not be written to %s: %w", second, BusAccountsState, err) } v.saidHashed(second, secret, c.Generation, fp, at) marked := *curKey marked.Rotating = &Rotating{Login: second, Generation: c.Generation, Since: at, Revision: revision} if err := v.Accounts.Put(marked); err != nil { return NodeLogin{}, fmt.Errorf("%s could not be marked rotating: %w", cur, err) } nl := NodeLogin{Node: node, Login: cur, Rotating: second, Stage: StageBegun, Since: at, ListRevision: revision} if err := v.putLogin(nl); err != nil { return NodeLogin{}, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err) } v.sayRotation(RotationBegunEvent, nl, map[string]any{"why": why, "act": act, "by": by}) v.log("the rotation of %s begins at the loaded list's revision %d: %s alternates to %s (asked by %s, hand act %s)", node, revision, cur, second, by, act) return nl, nil } // sayRotation says an event of a rotation: the node, its logins, the stage; never a value or a hash. func (v *Vault) sayRotation(event string, nl NodeLogin, extra map[string]any) { if v.Emit == nil { return } body := map[string]any{"node": nl.Node, "login": nl.Login, "rotating": nl.Rotating, "stage": nl.Stage, "at": v.Now().UTC().Format(time.RFC3339Nano)} for k, x := range extra { body[k] = x } if err := v.Emit(event, body); err != nil { v.log("the rotation of %s moved, but %s could not be said: %v", nl.Node, event, err) } } // loadedListHeld keeps the users the bus node's loaded list holds, with their hashes (what its handing opened to). func (v *Vault) loadedListHeld(from string, users map[string]string) { v.nr.mu.Lock() v.nr.loaded, v.nr.loadedFrom = users, from v.nr.mu.Unlock() } // loadedHolds says the loaded list holds user with exactly hash; loadedLacks that it is known and holds no user. func (v *Vault) loadedHolds(user, hash string) bool { v.nr.mu.Lock() defer v.nr.mu.Unlock() return v.nr.loaded != nil && hash != "" && v.nr.loaded[user] == hash } func (v *Vault) loadedLacks(user string) bool { v.nr.mu.Lock() defer v.nr.mu.Unlock() if len(v.nr.loaded) == 0 { return false } _, held := v.nr.loaded[user] return !held } // advanceRotations moves every rotation as far as the loaded list lets it: begun to ready when the list holds the new // login, confirmed to done when it lacks the old one. At each adoption of the loaded list and at the minute pass. func (v *Vault) advanceRotations() { running, err := v.rotatingNodes() if err != nil { v.log("the rotations could not be read: %v", err) return } for _, nl := range running { switch nl.Stage { case StageBegun: k, err := v.Accounts.Get(nl.Rotating) if err != nil || k == nil || !v.loadedHolds(nl.Rotating, k.Hash) { continue } nl.Stage = StageReady if err := v.putLogin(nl); err != nil { v.log("%s is ready, and %s could not be written: %v", nl.Rotating, NodeLoginsState, err) continue } v.sayRotation(NodeAccountReadyEvent, nl, nil) v.log("the loaded user list holds %s: %s may dial as it", nl.Rotating, nl.Node) case StageConfirmed: if !v.loadedLacks(nl.Rotating) { continue } v.finishRotation(nl) } } } // finishRotation ends a confirmed rotation whose old hash the loaded list lacks: the old value dropped, the rotating part // of the key cleared, rotation-done said. func (v *Vault) finishRotation(nl NodeLogin) { old := nl.Rotating if err := v.Store.Delete(loginSecret(nl.Node, old)); err != nil { v.log("the old login %s of %s is out of the list, and its value could not be dropped: %v", old, nl.Node, err) return } done := NodeLogin{Node: nl.Node, Login: nl.Login, Since: v.Now().UTC()} if err := v.putLogin(done); err != nil { v.log("the rotation of %s is done, and %s could not be written: %v", nl.Node, NodeLoginsState, err) return } v.sayRotation(RotationDoneEvent, done, map[string]any{"dropped": old}) v.log("the rotation of %s is done: it dials as %s, and %s is gone from the list and the vault", nl.Node, nl.Login, old) } // ConfirmAnswer is what `node-account-confirm` answers. type ConfirmAnswer struct { Node string `json:"node"` Login string `json:"login"` Stage string `json:"stage"` } // NodeAccountConfirm answers `node-account-confirm`: the node-engine, dialled as the new login (which the runtime // proves), says it connects with it. The old login's hash is then removed from bus-accounts; rotation-done follows when // the loaded list lacks it. func (v *Vault) NodeAccountConfirm(caller string) (ConfirmAnswer, error) { if err := fromAGrantedCaller(caller, "node-account-confirm"); err != nil { return ConfirmAnswer{}, err } node, ok := loginNode(caller) if !ok { return ConfirmAnswer{}, fmt.Errorf("%s is no node-engine's login: nothing is retired", caller) } nl, err := v.getLogin(node) if err != nil { return ConfirmAnswer{}, err } if nl == nil || nl.Rotating != caller || (nl.Stage != StageBegun && nl.Stage != StageReady) { return ConfirmAnswer{}, fmt.Errorf("%s is not the new login of a rotation of %s under way: nothing is retired", caller, node) } old := nl.Login if err := v.Accounts.Delete(old); err != nil { return ConfirmAnswer{}, fmt.Errorf("the hash of %s could not be taken out of %s: %w", old, BusAccountsState, err) } confirmed := NodeLogin{Node: node, Login: caller, Rotating: old, Stage: StageConfirmed, Since: v.Now().UTC(), ListRevision: nl.ListRevision} if err := v.putLogin(confirmed); err != nil { return ConfirmAnswer{}, fmt.Errorf("the old hash of %s is out, and %s %s could not be written: %w", node, NodeLoginsState, node, err) } v.log("%s confirmed %s: the hash of %s is removed from %s", node, caller, old, BusAccountsState) v.advanceRotations() return ConfirmAnswer{Node: node, Login: caller, Stage: StageConfirmed}, nil } // AbandonAnswer says what `abandon` took away. type AbandonAnswer struct { Node string `json:"node"` Login string `json:"login,omitempty"` Dropped string `json:"dropped,omitempty"` Queued bool `json:"was_queued,omitempty"` } // AbandonRotation answers `abandon`: the new login and its value are dropped, the old hash kept and the rotating part // cleared; a rotation only asked is taken off the queue. Not after the confirmation, when the old hash is already gone. func (v *Vault) AbandonRotation(caller, node string) (AbandonAnswer, error) { if err := fromAGrantedCaller(caller, "abandon"); err != nil { return AbandonAnswer{}, err } if err := enrolNode(node); err != nil { return AbandonAnswer{}, err } v.nr.mu.Lock() defer v.nr.mu.Unlock() out := AbandonAnswer{Node: node} if i := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == node }); i >= 0 { v.nr.queue = slices.Delete(v.nr.queue, i, i+1) out.Queued = true } nl, err := v.getLogin(node) if err != nil { return out, err } if nl == nil || nl.Rotating == "" { if out.Queued { return out, nil } return out, fmt.Errorf("%s rotates nothing: nothing was abandoned", node) } if nl.Stage == StageConfirmed { return out, fmt.Errorf("the rotation of %s is confirmed: the old login %s is out of the list, and %s is the login in use; rotate again to change it", node, nl.Rotating, nl.Login) } second := nl.Rotating if err := v.Accounts.Delete(second); err != nil { return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", second, BusAccountsState, err) } if err := v.Store.Delete(loginSecret(node, second)); err != nil { return out, fmt.Errorf("the value of %s could not be dropped: %w", second, err) } if k, err := v.Accounts.Get(nl.Login); err == nil && k != nil && k.Rotating != nil { cleared := *k cleared.Rotating = nil if err := v.Accounts.Put(cleared); err != nil { return out, fmt.Errorf("%s could not be marked as not rotating: %w", nl.Login, err) } } if err := v.putLogin(NodeLogin{Node: node, Login: nl.Login, Since: v.Now().UTC()}); err != nil { return out, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err) } _ = v.recordHanding(Handing{To: caller, Name: loginSecret(node, second), Outcome: "dropped", Why: "rotation of " + node + " abandoned"}) v.log("the rotation of %s is abandoned by %s: %s and its value are dropped, %s keeps its hash", node, caller, second, nl.Login) out.Login, out.Dropped = nl.Login, second return out, nil } // RotationPass is the minute pass of rotations: each moves as far as the loaded list lets it, and the next queued one // begins when none runs (the bus node after every other). func (v *Vault) RotationPass() { if v.Logins == nil || v.Accounts == nil { return } v.advanceRotations() running, err := v.rotatingNodes() if err != nil || len(running) > 0 { return } bus, err := v.busNode() if err != nil { return } v.minting.mu.Lock() revision := v.minting.loadedRevision v.minting.mu.Unlock() if revision == 0 { return } v.nr.mu.Lock() defer v.nr.mu.Unlock() next := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node != bus }) if next < 0 { next = slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == bus }) } if next < 0 { return } q := v.nr.queue[next] v.nr.queue = slices.Delete(v.nr.queue, next, next+1) if _, err := v.beginNodeRotation(q.By, q.Node, q.Why, q.Act, revision); err != nil { v.log("the queued rotation of %s did not begin: %v", q.Node, err) } } // RotationState is what vault_status says of rotations: the keys of node-logins that rotate, and the queue. type RotationState struct { Rotating []NodeLogin `json:"rotating"` Queued []queuedRotation `json:"queued"` } // RotationState answers how the rotations stand. func (v *Vault) RotationState() RotationState { running, _ := v.rotatingNodes() v.nr.mu.Lock() defer v.nr.mu.Unlock() return RotationState{Rotating: append([]NodeLogin{}, running...), Queued: append([]queuedRotation{}, v.nr.queue...)} } // LoginsPass deletes the key of every node that left the placement (ADR 0583 §1 insight). A placement that cannot be // read, or holds no node, deletes nothing. func (v *Vault) LoginsPass() { if v.Logins == nil || v.Placement.Keys == nil { return } placed, err := v.placements() if err != nil || len(placed) == 0 { return } keys, err := v.Logins.Keys() if err != nil { return } for _, k := range keys { if _, in := placed[k]; !in { v.nodeLeft(k) } } } // NodeLeft is heard when a node's key of the placement is deleted: its key of node-logins goes with it. func (v *Vault) NodeLeft(node string) { if v.Logins == nil || v.Placement.Read == nil { return } if _, _, found, err := v.nodePlaced(node); err != nil || found { return } v.nodeLeft(node) } func (v *Vault) nodeLeft(node string) { if err := v.Logins.Delete(node); err != nil { v.log("%s left the placement, and its key of %s could not be deleted: %v", node, NodeLoginsState, err) return } v.log("%s left the placement: its key of %s is deleted", node, NodeLoginsState) }