mesh-catalog/modules/mesh-vault/cmd/mesh-vault/noderotation.go

package main

// A node's two logins and their rotation (novox/hq ADR 0583 §1 to §7 and its insights; issue 774).
//
// A node-engine dials the bus as `node.<n>` or as `node.<n>--b`, the two logins alternating. The vault's state
// `node-logins` holds one key per node, `<n>`: the login in use, and while a rotation runs the other login with its stage
// (begun, ready, confirmed). Each node-engine reads its own key alone (the bus grants it that read); the vault writes it
// alone.
//
// `rotate-node` begins a rotation: answered only for a call that carries a hand act recorded through mesh-hand-acts, read
// back by its id; one rotation at a time mesh-wide, a call asked meanwhile is queued and begun at the pass that finds the
// first done; the bus node last, after every other node asked. It makes the other login's value, keeps it as
// `own/<n>/mesh-host/<login>` (sealed, with the operator copy; the node-engine takes it through `own`, sealed to its own
// key), writes its hash as the key `<login>` of bus-accounts and marks the login in use as rotating, so the grants part
// composes both. `node-account-ready` is said only when the user list the bus node loaded holds the new login's hash.
// `node-account-confirm` is the node-engine's, dialled as the new login, which the runtime proves: the old login's hash
// is then removed, and `rotation-done` is said once the loaded list lacks it, when the old value is dropped and the
// rotating part of the key cleared. `abandon` drops the new login and its value before the confirmation and keeps the
// old hash. A key whose node leaves the placement is deleted.

import (
	"encoding/json"
	"errors"
	"fmt"
	"slices"
	"sort"
	"strings"
	"sync"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"golang.org/x/crypto/bcrypt"
)

// The events a rotation says and the stages of a node's key are mesh-sdk's (delivery/node_logins.go); a rotation warranted
// by a hand act is warranted for a day.
const (
	RotationBegunEvent    = delivery.RotationBegunEvent
	NodeAccountReadyEvent = delivery.NodeAccountReadyEvent
	RotationDoneEvent     = delivery.RotationDoneEvent

	StageBegun     = delivery.RotationBegun
	StageReady     = delivery.RotationReady
	StageConfirmed = delivery.RotationConfirmed

	// HandActWithin is how recent a hand act is that warrants a rotation.
	HandActWithin = 24 * time.Hour
)

// NodeLogin is one key of the vault's state node-logins (mesh-sdk's contract): the login the node's node-engine dials as,
// and while a rotation runs the other login with its stage. Never a value or a hash.
type NodeLogin = delivery.NodeLogin

// NodeLoginStore is the vault's state `node-logins`, by node.
type NodeLoginStore interface {
	Get(node string) (*NodeLogin, error)
	Put(n NodeLogin) error
	Delete(node string) error
	Keys() ([]string, error)
}

// busNodeLogins is NodeLoginStore over the vault's own state through its runtime.
type busNodeLogins struct{}

func (busNodeLogins) Get(node string) (*NodeLogin, error) {
	e, err := stdio.State(NodeLoginsState).Get(node)
	if err != nil || e == nil {
		return nil, err
	}
	n, err := delivery.DecodeNodeLogin(e.Value)
	if err != nil {
		return nil, fmt.Errorf("%s %s is not a node's login: %w", NodeLoginsState, node, err)
	}
	return &n, nil
}

func (busNodeLogins) Put(n NodeLogin) error {
	raw, err := delivery.EncodeNodeLogin(n)
	if err != nil {
		return err
	}
	_, err = stdio.State(NodeLoginsState).Put(n.Node, json.RawMessage(raw))
	return err
}

func (busNodeLogins) Delete(node string) error { return stdio.State(NodeLoginsState).Delete(node) }

func (busNodeLogins) Keys() ([]string, error) { return stdio.State(NodeLoginsState).Keys() }

// HandActSeen is a hand act as the vault reads it back from mesh-hand-acts by its id: who recorded it and when.
type HandActSeen struct {
	ID string    `json:"id"`
	By string    `json:"by"`
	At time.Time `json:"at"`
}

// ActReader reads a hand act back by its id: nil when mesh-hand-acts holds none.
type ActReader func(id string) (*HandActSeen, error)

// queuedRotation is a `rotate-node` asked while another ran, begun at the pass that finds none running.
type queuedRotation struct {
	Node string    `json:"node"`
	By   string    `json:"by"`
	Why  string    `json:"why"`
	Act  string    `json:"act"`
	At   time.Time `json:"at"`
}

// nodeRotation is what the vault keeps in memory of its rotations: the user list the bus node loaded (user to hash), and
// the rotations queued. The record is node-logins and bus-accounts; a restart forgets the queue, and the one who asked
// asks again.
type nodeRotation struct {
	mu         sync.Mutex
	loaded     map[string]string
	loadedFrom string
	queue      []queuedRotation
}

func (v *Vault) getLogin(node string) (*NodeLogin, error) {
	if v.Logins == nil {
		return nil, errors.New("the vault has no state " + NodeLoginsState + " to read")
	}
	return v.Logins.Get(node)
}

func (v *Vault) putLogin(n NodeLogin) error {
	if v.Logins == nil {
		return errors.New("the vault has no state " + NodeLoginsState + " to write")
	}
	return v.Logins.Put(n)
}

func (v *Vault) deleteLogin(node string) error {
	if v.Logins == nil {
		return nil
	}
	return v.Logins.Delete(node)
}

// currentLogin is the login node dials as: its key's, or `node.<node>` before any key is written.
func (v *Vault) currentLogin(node string) (string, error) {
	nl, err := v.getLogin(node)
	if err != nil {
		return "", err
	}
	if nl != nil && nl.Login != "" {
		return nl.Login, nil
	}
	return NodeEnginePrefix + node, nil
}

// otherLogin is the login a node alternates to from login.
func otherLogin(login string) string {
	if base, second := strings.CutSuffix(login, SecondLoginSuffix); second {
		return base
	}
	return login + SecondLoginSuffix
}

// loginNode is the node whose login a bus user is: `node.<n>` or `node.<n>--b`.
func loginNode(user string) (string, bool) {
	rest, ok := strings.CutPrefix(user, NodeEnginePrefix)
	if !ok {
		return "", false
	}
	node := strings.TrimSuffix(rest, SecondLoginSuffix)
	return node, node != "" && enrolNode(node) == nil
}

// checkAct reads the hand act back by its id and refuses a rotation it does not warrant: none recorded, recorded by
// another caller, or older than a day.
func (v *Vault) checkAct(caller, id string) error {
	if strings.TrimSpace(id) == "" {
		return errors.New("rotate-node carries the hand act that warrants it (`act`): record it through mesh-hand-acts first (novox/hq ADR 0583 §2)")
	}
	if v.ReadAct == nil {
		return errors.New("the vault reads no hand acts, so it answers no rotation (novox/hq ADR 0583 §2)")
	}
	act, err := v.ReadAct(id)
	if err != nil {
		return fmt.Errorf("the hand act %s could not be read back from mesh-hand-acts: %w", id, err)
	}
	switch {
	case act == nil:
		return fmt.Errorf("mesh-hand-acts holds no hand act %s: record one first (novox/hq ADR 0583 §2)", id)
	case act.By != caller:
		return fmt.Errorf("the hand act %s was recorded by %s, not by %s who asks: a rotation is warranted by its own caller's act", id, act.By, caller)
	case v.Now().Sub(act.At) > HandActWithin:
		return fmt.Errorf("the hand act %s is older than %s: record a new one", id, HandActWithin)
	}
	return nil
}

// rotatingNodes are the nodes whose key says a rotation runs, sorted.
func (v *Vault) rotatingNodes() ([]NodeLogin, error) {
	if v.Logins == nil {
		return nil, nil
	}
	keys, err := v.Logins.Keys()
	if err != nil {
		return nil, err
	}
	var out []NodeLogin
	for _, k := range keys {
		nl, err := v.Logins.Get(k)
		if err != nil {
			return nil, err
		}
		if nl != nil && nl.Rotating != "" {
			out = append(out, *nl)
		}
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
	return out, nil
}

// RotateNodeArgs are `rotate-node`'s arguments.
type RotateNodeArgs struct {
	Node string
	Why  string
	Act  string
}

// RotateNodeAnswer says a rotation began, or that it waits for the ones before it. Never a value.
type RotateNodeAnswer struct {
	Node     string `json:"node"`
	Login    string `json:"login,omitempty"`
	Rotating string `json:"rotating,omitempty"`
	Stage    string `json:"stage,omitempty"`
	Revision uint64 `json:"list_revision,omitempty"`
	Queued   bool   `json:"queued,omitempty"`
	// Behind are the nodes whose rotation it waits for, running or asked before.
	Behind []string `json:"behind,omitempty"`
}

// RotateNode answers caller's `rotate-node`.
func (v *Vault) RotateNode(caller string, a RotateNodeArgs) (RotateNodeAnswer, error) {
	if err := fromAGrantedCaller(caller, "rotate-node"); err != nil {
		return RotateNodeAnswer{}, err
	}
	if err := enrolNode(a.Node); err != nil {
		return RotateNodeAnswer{}, err
	}
	if strings.TrimSpace(a.Why) == "" {
		return RotateNodeAnswer{}, errors.New("rotate-node says why (`why`)")
	}
	if err := v.checkAct(caller, a.Act); err != nil {
		return RotateNodeAnswer{}, err
	}
	if v.Accounts == nil {
		return RotateNodeAnswer{}, errors.New("the vault has no state of bus accounts to write to")
	}
	v.leaseAfresh()
	if on, why := v.leaseHanded(); !on {
		return RotateNodeAnswer{}, fmt.Errorf("no rotation begins: %s", why)
	}
	v.minting.mu.Lock()
	revision := v.minting.loadedRevision
	v.minting.mu.Unlock()
	if revision == 0 {
		return RotateNodeAnswer{}, errors.New("no rotation begins: the revision of the user list the bus loaded is not known yet")
	}
	bus, err := v.busNode()
	if err != nil {
		return RotateNodeAnswer{}, fmt.Errorf("no rotation is ordered: the node holding the bus is not known (the bus node rotates last): %w", err)
	}
	v.nr.mu.Lock()
	defer v.nr.mu.Unlock()
	if nl, err := v.getLogin(a.Node); err != nil {
		return RotateNodeAnswer{}, err
	} else if nl != nil && nl.Rotating != "" {
		return RotateNodeAnswer{}, fmt.Errorf("%s is rotating already, since %s (second login %s)", a.Node, nl.Since.Format(time.RFC3339), nl.Rotating)
	}
	for _, q := range v.nr.queue {
		if q.Node == a.Node {
			return RotateNodeAnswer{}, fmt.Errorf("%s's rotation is asked already, by %s, and waits its turn", a.Node, q.By)
		}
	}
	running, err := v.rotatingNodes()
	if err != nil {
		return RotateNodeAnswer{}, err
	}
	var behind []string
	for _, r := range running {
		behind = append(behind, r.Node)
	}
	if a.Node == bus {
		for _, q := range v.nr.queue {
			behind = append(behind, q.Node)
		}
	}
	if len(behind) > 0 {
		v.nr.queue = append(v.nr.queue, queuedRotation{Node: a.Node, By: caller, Why: a.Why, Act: a.Act, At: v.Now().UTC()})
		v.log("the rotation of %s waits for %s (asked by %s): one rotation at a time, the bus node %s last", a.Node,
			strings.Join(behind, ", "), caller, bus)
		return RotateNodeAnswer{Node: a.Node, Queued: true, Behind: behind}, nil
	}
	nl, err := v.beginNodeRotation(caller, a.Node, a.Why, a.Act, revision)
	if err != nil {
		return RotateNodeAnswer{}, err
	}
	return RotateNodeAnswer{Node: a.Node, Login: nl.Login, Rotating: nl.Rotating, Stage: nl.Stage, Revision: revision}, nil
}

// beginNodeRotation makes the other login and records the rotation; under nr.mu.
func (v *Vault) beginNodeRotation(by, node, why, act string, revision uint64) (NodeLogin, error) {
	cur, err := v.currentLogin(node)
	if err != nil {
		return NodeLogin{}, err
	}
	curKey, err := v.Accounts.Get(cur)
	if err != nil {
		return NodeLogin{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, cur, err)
	}
	if curKey == nil {
		return NodeLogin{}, fmt.Errorf("%s holds no key of %s, the login %s dials as: there is nothing to rotate", BusAccountsState, cur, node)
	}
	second := otherLogin(cur)
	value, password, err := v.accountValue(node, second)
	if err != nil {
		return NodeLogin{}, err
	}
	secret := loginSecret(node, second)
	c, err := v.keep(secret, value, "the vault", "", OriginMade)
	if err != nil {
		return NodeLogin{}, fmt.Errorf("the login %s is not kept: %w", second, err)
	}
	hash, err := bcrypt.GenerateFromPassword([]byte(password), v.hashCost())
	if err != nil {
		return NodeLogin{}, err
	}
	at := v.Now().UTC()
	fp := Fingerprint(value)
	if err := v.Accounts.Put(AccountHash{User: second, Hash: string(hash), Of: secret, Generation: c.Generation,
		Fingerprint: fp, At: at, Kept: KeptNode}); err != nil {
		return NodeLogin{}, fmt.Errorf("the hash of %s could not be written to %s: %w", second, BusAccountsState, err)
	}
	v.saidHashed(second, secret, c.Generation, fp, at)
	marked := *curKey
	marked.Rotating = &Rotating{Login: second, Generation: c.Generation, Since: at, Revision: revision}
	if err := v.Accounts.Put(marked); err != nil {
		return NodeLogin{}, fmt.Errorf("%s could not be marked rotating: %w", cur, err)
	}
	nl := NodeLogin{Node: node, Login: cur, Rotating: second, Stage: StageBegun, Since: at, ListRevision: revision}
	if err := v.putLogin(nl); err != nil {
		return NodeLogin{}, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err)
	}
	v.sayRotation(RotationBegunEvent, nl, map[string]any{"why": why, "act": act, "by": by})
	v.log("the rotation of %s begins at the loaded list's revision %d: %s alternates to %s (asked by %s, hand act %s)", node,
		revision, cur, second, by, act)
	return nl, nil
}

// sayRotation says an event of a rotation: the node, its logins, the stage; never a value or a hash.
func (v *Vault) sayRotation(event string, nl NodeLogin, extra map[string]any) {
	if v.Emit == nil {
		return
	}
	body := map[string]any{"node": nl.Node, "login": nl.Login, "rotating": nl.Rotating, "stage": nl.Stage,
		"at": v.Now().UTC().Format(time.RFC3339Nano)}
	for k, x := range extra {
		body[k] = x
	}
	if err := v.Emit(event, body); err != nil {
		v.log("the rotation of %s moved, but %s could not be said: %v", nl.Node, event, err)
	}
}

// loadedListHeld keeps the users the bus node's loaded list holds, with their hashes (what its handing opened to).
func (v *Vault) loadedListHeld(from string, users map[string]string) {
	v.nr.mu.Lock()
	v.nr.loaded, v.nr.loadedFrom = users, from
	v.nr.mu.Unlock()
}

// loadedHolds says the loaded list holds user with exactly hash; loadedLacks that it is known and holds no user.
func (v *Vault) loadedHolds(user, hash string) bool {
	v.nr.mu.Lock()
	defer v.nr.mu.Unlock()
	return v.nr.loaded != nil && hash != "" && v.nr.loaded[user] == hash
}

func (v *Vault) loadedLacks(user string) bool {
	v.nr.mu.Lock()
	defer v.nr.mu.Unlock()
	if len(v.nr.loaded) == 0 {
		return false
	}
	_, held := v.nr.loaded[user]
	return !held
}

// advanceRotations moves every rotation as far as the loaded list lets it: begun to ready when the list holds the new
// login, confirmed to done when it lacks the old one. At each adoption of the loaded list and at the minute pass.
func (v *Vault) advanceRotations() {
	running, err := v.rotatingNodes()
	if err != nil {
		v.log("the rotations could not be read: %v", err)
		return
	}
	for _, nl := range running {
		switch nl.Stage {
		case StageBegun:
			k, err := v.Accounts.Get(nl.Rotating)
			if err != nil || k == nil || !v.loadedHolds(nl.Rotating, k.Hash) {
				continue
			}
			nl.Stage = StageReady
			if err := v.putLogin(nl); err != nil {
				v.log("%s is ready, and %s could not be written: %v", nl.Rotating, NodeLoginsState, err)
				continue
			}
			v.sayRotation(NodeAccountReadyEvent, nl, nil)
			v.log("the loaded user list holds %s: %s may dial as it", nl.Rotating, nl.Node)
		case StageConfirmed:
			if !v.loadedLacks(nl.Rotating) {
				continue
			}
			v.finishRotation(nl)
		}
	}
}

// finishRotation ends a confirmed rotation whose old hash the loaded list lacks: the old value dropped, the rotating part
// of the key cleared, rotation-done said.
func (v *Vault) finishRotation(nl NodeLogin) {
	old := nl.Rotating
	if err := v.Store.Delete(loginSecret(nl.Node, old)); err != nil {
		v.log("the old login %s of %s is out of the list, and its value could not be dropped: %v", old, nl.Node, err)
		return
	}
	done := NodeLogin{Node: nl.Node, Login: nl.Login, Since: v.Now().UTC()}
	if err := v.putLogin(done); err != nil {
		v.log("the rotation of %s is done, and %s could not be written: %v", nl.Node, NodeLoginsState, err)
		return
	}
	v.sayRotation(RotationDoneEvent, done, map[string]any{"dropped": old})
	v.log("the rotation of %s is done: it dials as %s, and %s is gone from the list and the vault", nl.Node, nl.Login, old)
}

// ConfirmAnswer is what `node-account-confirm` answers.
type ConfirmAnswer struct {
	Node  string `json:"node"`
	Login string `json:"login"`
	Stage string `json:"stage"`
}

// NodeAccountConfirm answers `node-account-confirm`: the node-engine, dialled as the new login (which the runtime
// proves), says it connects with it. The old login's hash is then removed from bus-accounts; rotation-done follows when
// the loaded list lacks it.
func (v *Vault) NodeAccountConfirm(caller string) (ConfirmAnswer, error) {
	if err := fromAGrantedCaller(caller, "node-account-confirm"); err != nil {
		return ConfirmAnswer{}, err
	}
	node, ok := loginNode(caller)
	if !ok {
		return ConfirmAnswer{}, fmt.Errorf("%s is no node-engine's login: nothing is retired", caller)
	}
	nl, err := v.getLogin(node)
	if err != nil {
		return ConfirmAnswer{}, err
	}
	if nl == nil || nl.Rotating != caller || (nl.Stage != StageBegun && nl.Stage != StageReady) {
		return ConfirmAnswer{}, fmt.Errorf("%s is not the new login of a rotation of %s under way: nothing is retired", caller, node)
	}
	old := nl.Login
	if err := v.Accounts.Delete(old); err != nil {
		return ConfirmAnswer{}, fmt.Errorf("the hash of %s could not be taken out of %s: %w", old, BusAccountsState, err)
	}
	confirmed := NodeLogin{Node: node, Login: caller, Rotating: old, Stage: StageConfirmed, Since: v.Now().UTC(), ListRevision: nl.ListRevision}
	if err := v.putLogin(confirmed); err != nil {
		return ConfirmAnswer{}, fmt.Errorf("the old hash of %s is out, and %s %s could not be written: %w", node, NodeLoginsState, node, err)
	}
	v.log("%s confirmed %s: the hash of %s is removed from %s", node, caller, old, BusAccountsState)
	v.advanceRotations()
	return ConfirmAnswer{Node: node, Login: caller, Stage: StageConfirmed}, nil
}

// AbandonAnswer says what `abandon` took away.
type AbandonAnswer struct {
	Node    string `json:"node"`
	Login   string `json:"login,omitempty"`
	Dropped string `json:"dropped,omitempty"`
	Queued  bool   `json:"was_queued,omitempty"`
}

// AbandonRotation answers `abandon`: the new login and its value are dropped, the old hash kept and the rotating part
// cleared; a rotation only asked is taken off the queue. Not after the confirmation, when the old hash is already gone.
func (v *Vault) AbandonRotation(caller, node string) (AbandonAnswer, error) {
	if err := fromAGrantedCaller(caller, "abandon"); err != nil {
		return AbandonAnswer{}, err
	}
	if err := enrolNode(node); err != nil {
		return AbandonAnswer{}, err
	}
	v.nr.mu.Lock()
	defer v.nr.mu.Unlock()
	out := AbandonAnswer{Node: node}
	if i := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == node }); i >= 0 {
		v.nr.queue = slices.Delete(v.nr.queue, i, i+1)
		out.Queued = true
	}
	nl, err := v.getLogin(node)
	if err != nil {
		return out, err
	}
	if nl == nil || nl.Rotating == "" {
		if out.Queued {
			return out, nil
		}
		return out, fmt.Errorf("%s rotates nothing: nothing was abandoned", node)
	}
	if nl.Stage == StageConfirmed {
		return out, fmt.Errorf("the rotation of %s is confirmed: the old login %s is out of the list, and %s is the login in use; rotate again to change it",
			node, nl.Rotating, nl.Login)
	}
	second := nl.Rotating
	if err := v.Accounts.Delete(second); err != nil {
		return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", second, BusAccountsState, err)
	}
	if err := v.Store.Delete(loginSecret(node, second)); err != nil {
		return out, fmt.Errorf("the value of %s could not be dropped: %w", second, err)
	}
	if k, err := v.Accounts.Get(nl.Login); err == nil && k != nil && k.Rotating != nil {
		cleared := *k
		cleared.Rotating = nil
		if err := v.Accounts.Put(cleared); err != nil {
			return out, fmt.Errorf("%s could not be marked as not rotating: %w", nl.Login, err)
		}
	}
	if err := v.putLogin(NodeLogin{Node: node, Login: nl.Login, Since: v.Now().UTC()}); err != nil {
		return out, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err)
	}
	_ = v.recordHanding(Handing{To: caller, Name: loginSecret(node, second), Outcome: "dropped", Why: "rotation of " + node + " abandoned"})
	v.log("the rotation of %s is abandoned by %s: %s and its value are dropped, %s keeps its hash", node, caller, second, nl.Login)
	out.Login, out.Dropped = nl.Login, second
	return out, nil
}

// RotationPass is the minute pass of rotations: each moves as far as the loaded list lets it, and the next queued one
// begins when none runs (the bus node after every other).
func (v *Vault) RotationPass() {
	if v.Logins == nil || v.Accounts == nil {
		return
	}
	v.advanceRotations()
	running, err := v.rotatingNodes()
	if err != nil || len(running) > 0 {
		return
	}
	v.leaseAfresh()
	if on, _ := v.leaseHanded(); !on {
		return
	}
	bus, err := v.busNode()
	if err != nil {
		return
	}
	v.minting.mu.Lock()
	revision := v.minting.loadedRevision
	v.minting.mu.Unlock()
	if revision == 0 {
		return
	}
	v.nr.mu.Lock()
	defer v.nr.mu.Unlock()
	next := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node != bus })
	if next < 0 {
		next = slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == bus })
	}
	if next < 0 {
		return
	}
	q := v.nr.queue[next]
	v.nr.queue = slices.Delete(v.nr.queue, next, next+1)
	if _, err := v.beginNodeRotation(q.By, q.Node, q.Why, q.Act, revision); err != nil {
		v.log("the queued rotation of %s did not begin: %v", q.Node, err)
	}
}

// RotationState is what vault_status says of rotations: the keys of node-logins that rotate, and the queue.
type RotationState struct {
	Rotating []NodeLogin      `json:"rotating"`
	Queued   []queuedRotation `json:"queued"`
}

// RotationState answers how the rotations stand.
func (v *Vault) RotationState() RotationState {
	running, _ := v.rotatingNodes()
	v.nr.mu.Lock()
	defer v.nr.mu.Unlock()
	return RotationState{Rotating: append([]NodeLogin{}, running...), Queued: append([]queuedRotation{}, v.nr.queue...)}
}

// LoginsPass deletes the key of every node that left the placement (ADR 0583 §1 insight). A placement that cannot be
// read, or holds no node, deletes nothing.
func (v *Vault) LoginsPass() {
	if v.Logins == nil || v.Placement.Keys == nil {
		return
	}
	placed, err := v.placements()
	if err != nil || len(placed) == 0 {
		return
	}
	keys, err := v.Logins.Keys()
	if err != nil {
		return
	}
	for _, k := range keys {
		if _, in := placed[k]; !in {
			v.nodeLeft(k)
		}
	}
}

// NodeLeft is heard when a node's key of the placement is deleted: its key of node-logins goes with it.
func (v *Vault) NodeLeft(node string) {
	if v.Logins == nil || v.Placement.Read == nil {
		return
	}
	if _, _, found, err := v.nodePlaced(node); err != nil || found {
		return
	}
	v.nodeLeft(node)
}

func (v *Vault) nodeLeft(node string) {
	if err := v.Logins.Delete(node); err != nil {
		v.log("%s left the placement, and its key of %s could not be deleted: %v", node, NodeLoginsState, err)
		return
	}
	v.log("%s left the placement: its key of %s is deleted", node, NodeLoginsState)
}