mesh-tools
97a2d841c7
The controller plans config, templates and moved providers' provisions, hands bus accounts to the vault and judges the new manifest words, on the feature's SDK and node-engine validator (feature 57, config and provisioning).
337 lines
12 KiB
Go
337 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// `secret move` — the controller's secrets move into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676;
|
|
// the operator's decision of 2026-10-14 on that issue). Run once, after the feature's delivery.
|
|
//
|
|
// **The controller cannot open what it holds, so each machine moves its own.** For every machine — or the ones named —
|
|
// the controller reads every row sealed to it, as the store holds it, and asks that machine's node-engine to move each
|
|
// one, signed (link.AskMoveSecret): the engine opens it with its sealing key and hands it to the vault's move-in, sealed
|
|
// to the call's key, keeping its origin and generation. The vault takes it from that engine alone, never over a value
|
|
// it holds, and reads back what it kept to the engine, which compares; the engine answers only then. What moved is
|
|
// recorded here, never a value. Nothing is rotated, and nothing points at the vault because of this: a module's
|
|
// secrets are filled from the vault only once it moves (V4, issue 647).
|
|
//
|
|
// **A machine asleep is moved when it is next heard from**: once the move has started, a machine that reports is moved
|
|
// in the background (moveWhenHeard), and what is left of it is said on the next `secret move`.
|
|
//
|
|
// The controller's record of asks at a desk, which holds no value, is copied as it is, once (`move-asks`).
|
|
|
|
const secretMoveUsage = "secret move [<node>] — move every secret the controller holds into the vault, as it is: each " +
|
|
"machine's node-engine opens what was sealed to it and hands it to the vault, which reads it back; no value is " +
|
|
"rotated, and a machine not answering is moved when it is next heard from"
|
|
|
|
// moveAsk is the ask of one row of a machine's engine.
|
|
type moveAsk func(row inventory.SecretToMove, ask link.MoveSecretAsk) (link.MoveSecretAnswer, error)
|
|
|
|
// nodeMove is what the move did on one machine.
|
|
type nodeMove struct {
|
|
Node string
|
|
Moved int
|
|
Already int
|
|
// Refused are the rows not moved, each with why; Changed the rows changed in the store after they moved.
|
|
Refused []string
|
|
Changed []string
|
|
// Unreached is why the machine's engine could not be asked: it is moved when next heard from.
|
|
Unreached error
|
|
// Left is how many rows were not asked because the machine was not reached.
|
|
Left int
|
|
// Failed is why its rows could not be read.
|
|
Failed error
|
|
}
|
|
|
|
// done says nothing is left to move on the machine.
|
|
func (m nodeMove) done() bool {
|
|
return m.Failed == nil && m.Unreached == nil && len(m.Refused) == 0
|
|
}
|
|
|
|
func (m nodeMove) String() string {
|
|
var b strings.Builder
|
|
if m.Failed != nil {
|
|
fmt.Fprintf(&b, "%s: its rows could not be read, nothing was moved: %v\n", m.Node, m.Failed)
|
|
return b.String()
|
|
}
|
|
fmt.Fprintf(&b, "%s: %d moved into the vault and read back the same, %d there already", m.Node, m.Moved, m.Already)
|
|
if n := len(m.Refused); n > 0 {
|
|
fmt.Fprintf(&b, ", %d not moved", n)
|
|
}
|
|
b.WriteString("\n")
|
|
for _, r := range m.Refused {
|
|
fmt.Fprintf(&b, " not moved: %s\n", r)
|
|
}
|
|
for _, c := range m.Changed {
|
|
fmt.Fprintf(&b, " %s was changed in the controller's store after it moved; the vault keeps what it took, "+
|
|
"and the move never replaces it\n", c)
|
|
}
|
|
if m.Unreached != nil {
|
|
fmt.Fprintf(&b, " %d left: %s was not reached (%v); it is moved when it is next heard from\n", m.Left, m.Node,
|
|
m.Unreached)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// moveRowAsk is a row as its machine's engine is asked to move it.
|
|
func moveRowAsk(r inventory.SecretToMove) link.MoveSecretAsk {
|
|
a := link.MoveSecretAsk{Origin: r.Origin, Generation: r.Generation, MadeAt: r.MadeAt.UTC(), Sealed: r.Sealed,
|
|
SealedTo: r.SealedTo}
|
|
if r.Own() {
|
|
a.Module, a.Name = r.Module, r.Name
|
|
} else {
|
|
a.Provider, a.Provision, a.Consumer, a.ConsumerModule, a.Local = r.Provider, r.Provision, r.Consumer,
|
|
r.ConsumerModule, r.Local
|
|
}
|
|
return a
|
|
}
|
|
|
|
// moveNode moves every row sealed to one machine that has not moved, one ask each, and records what the vault kept.
|
|
// The first ask the engine does not answer ends the machine's turn: the rest wait for it to be heard from again.
|
|
func moveNode(node string, rows func(string) ([]inventory.SecretToMove, error), ask moveAsk,
|
|
record func(inventory.SecretToMove, int) error) nodeMove {
|
|
m := nodeMove{Node: node}
|
|
all, err := rows(node)
|
|
if err != nil {
|
|
m.Failed = err
|
|
return m
|
|
}
|
|
for n, r := range all {
|
|
switch {
|
|
case r.Moved != nil && r.Moved.Digest == r.Digest:
|
|
m.Already++
|
|
continue
|
|
case r.Moved != nil:
|
|
m.Changed = append(m.Changed, r.Secret)
|
|
continue
|
|
case !r.Current:
|
|
m.Refused = append(m.Refused, fmt.Sprintf("%s is sealed to an earlier sealing key of %s, which no longer "+
|
|
"opens it: nothing can move it; it moves once it is made or accepted again", r.Secret, node))
|
|
continue
|
|
}
|
|
answer, err := ask(r, moveRowAsk(r))
|
|
if err != nil {
|
|
m.Unreached, m.Left = err, len(all)-n
|
|
return m
|
|
}
|
|
if answer.Refused != "" {
|
|
m.Refused = append(m.Refused, fmt.Sprintf("%s: %s", r.Secret, answer.Refused))
|
|
continue
|
|
}
|
|
if answer.Secret != r.Secret {
|
|
m.Refused = append(m.Refused, fmt.Sprintf("%s: the vault kept it as %s, not under the name the move "+
|
|
"gives it", r.Secret, answer.Secret))
|
|
continue
|
|
}
|
|
if err := record(r, answer.Generation); err != nil {
|
|
m.Refused = append(m.Refused, fmt.Sprintf("%s is in the vault, but that it moved could not be recorded: %v",
|
|
r.Secret, err))
|
|
continue
|
|
}
|
|
if answer.Unchanged {
|
|
m.Already++
|
|
} else {
|
|
m.Moved++
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// movedAsksAtOnce is how many rows of the record of asks one call to the vault carries (mesh-vault MovedAsksAtOnce).
|
|
const movedAsksAtOnce = 500
|
|
|
|
// moveAsks copies the controller's record of asks into the vault, in parts, and answers how many it copied.
|
|
func moveAsks(asks []inventory.SecretAsk, vault vaultAsker) (int, error) {
|
|
copied := 0
|
|
for start := 0; start < len(asks); start += movedAsksAtOnce {
|
|
end := min(start+movedAsksAtOnce, len(asks))
|
|
var rows []map[string]any
|
|
for _, a := range asks[start:end] {
|
|
row := map[string]any{"id": a.ID, "node": a.Node, "module": a.Module, "name": a.Name, "desk": a.Desk,
|
|
"asked_by": a.AskedBy, "opened_at": a.OpenedAt.UTC().Format(time.RFC3339Nano)}
|
|
if a.EndedAt != nil {
|
|
row["ended_at"], row["outcome"] = a.EndedAt.UTC().Format(time.RFC3339Nano), a.Outcome
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
raw, err := vault("move-asks", map[string]any{"asks": rows})
|
|
if err != nil {
|
|
return copied, fmt.Errorf("the vault did not take the record of asks after %d rows: %w", copied, err)
|
|
}
|
|
var said struct {
|
|
Copied int `json:"copied"`
|
|
Already int `json:"already"`
|
|
}
|
|
_ = json.Unmarshal(raw, &said)
|
|
copied += said.Copied + said.Already
|
|
}
|
|
return copied, nil
|
|
}
|
|
|
|
// moveOnTheBus is the move's ask over the bus, signed with the mesh's key.
|
|
func moveOnTheBus(ctx context.Context, open *stores, conn *nats.Conn, by string) (moveAsk, error) {
|
|
ident, err := open.Identity(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked: %w", err)
|
|
}
|
|
return func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) {
|
|
return link.AskMoveSecret(ctx, conn, ident, r.Node, by, row, link.MoveSecretWithin)
|
|
}, nil
|
|
}
|
|
|
|
// secretMove is `secret move [<node>]`: the record of asks copied once, then every machine's rows — or one's — and
|
|
// what moved said per machine. Any row not moved fails the command, said by name.
|
|
func secretMove(ctx context.Context, args []string) error {
|
|
if len(args) > 1 || (len(args) == 1 && (args[0] == "" || strings.HasPrefix(args[0], "-"))) {
|
|
return errors.New(secretMoveUsage)
|
|
}
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
inv := open.inventory
|
|
by := actingAs()
|
|
if by == "" {
|
|
by = handOverBy()
|
|
}
|
|
move, err := inv.StartSecretMove(ctx, by)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var nodes []string
|
|
if len(args) == 1 {
|
|
if _, err := inv.NodeByName(ctx, args[0]); err != nil {
|
|
return fmt.Errorf("nothing was moved: %w", err)
|
|
}
|
|
nodes = args
|
|
} else {
|
|
all, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, n := range all {
|
|
nodes = append(nodes, n.Name)
|
|
}
|
|
}
|
|
fmt.Printf("the move of the secrets into the vault, started %s by %s\n", move.StartedAt.Local().Format("2006-01-02 15:04"),
|
|
move.StartedBy)
|
|
if move.AsksMovedAt == nil {
|
|
asks, err := inv.SecretAsks(ctx, time.Time{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
copied, err := moveAsks(asks, theVault(ctx))
|
|
if err != nil {
|
|
return fmt.Errorf("the record of asks was not copied, and no secret was moved: %w", err)
|
|
}
|
|
if err := inv.RecordAsksMoved(ctx); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("the record of asks at a desk: %d rows copied into the vault as they are\n", copied)
|
|
}
|
|
failed := 0
|
|
err = onTheBus(func(conn *nats.Conn) error {
|
|
ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+by)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, node := range nodes {
|
|
m := moveNode(node, func(n string) ([]inventory.SecretToMove, error) { return inv.SecretsToMoveFrom(ctx, n) },
|
|
ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) })
|
|
fmt.Print(m.String())
|
|
if m.Failed != nil || len(m.Refused) > 0 {
|
|
failed++
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if failed > 0 {
|
|
return fmt.Errorf("on %d machine(s) a secret was not moved, said above by name", failed)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// moveHeard is the machines being moved in the background now, and when each was last moved, so a machine that reports
|
|
// often is not asked again before moveHeardEvery has passed.
|
|
var moveHeard = struct {
|
|
sync.Mutex
|
|
running map[string]bool
|
|
last map[string]time.Time
|
|
}{running: map[string]bool{}, last: map[string]time.Time{}}
|
|
|
|
// moveHeardEvery is how often a machine heard from is moved at most.
|
|
const moveHeardEvery = 10 * time.Minute
|
|
|
|
// moveWhenHeard moves what is left of a machine that was just heard from, once the move has started: a machine asleep
|
|
// when it started is moved when it next reports. What it did is said on the controller's output, and recorded for the
|
|
// next `secret move`.
|
|
func moveWhenHeard(ctx context.Context, open *stores, conn *nats.Conn, node string) {
|
|
if open == nil || conn == nil || node == "" {
|
|
return
|
|
}
|
|
moveHeard.Lock()
|
|
if moveHeard.running[node] || time.Since(moveHeard.last[node]) < moveHeardEvery {
|
|
moveHeard.Unlock()
|
|
return
|
|
}
|
|
moveHeard.running[node], moveHeard.last[node] = true, time.Now()
|
|
moveHeard.Unlock()
|
|
defer func() {
|
|
moveHeard.Lock()
|
|
delete(moveHeard.running, node)
|
|
moveHeard.Unlock()
|
|
}()
|
|
inv := open.inventory
|
|
move, started, err := inv.SecretMoveStarted(ctx)
|
|
if err != nil || !started {
|
|
return
|
|
}
|
|
rows, err := inv.SecretsToMoveFrom(ctx, node)
|
|
if err != nil {
|
|
return
|
|
}
|
|
left := false
|
|
for _, r := range rows {
|
|
left = left || (r.Moved == nil && r.Current)
|
|
}
|
|
if !left {
|
|
return
|
|
}
|
|
ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+move.StartedBy+", on hearing from "+node)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from and its secrets not moved: %v\n", node, err)
|
|
return
|
|
}
|
|
m := moveNode(node, func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask,
|
|
func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) })
|
|
if m.done() {
|
|
fmt.Printf("mesh-controller: %s was heard from; the move of its secrets is done: %s", node, m.String())
|
|
return
|
|
}
|
|
fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from; the move of its secrets is not done: %s", node, m.String())
|
|
}
|
|
|
|
// moveHeardOn is the connection a machine heard from is moved over: the serving controller's bus.
|
|
func moveHeardOn() *nats.Conn {
|
|
if js := servingBus.Load(); js != nil {
|
|
return js.Conn()
|
|
}
|
|
return nil
|
|
}
|