package main import ( "context" "encoding/json" "errors" "fmt" "os" "strings" "sync" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // `secret move` — the controller's secrets move into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676; // the operator's decision of 2026-10-14 on that issue). Run once, after the feature's delivery. // // **The controller cannot open what it holds, so each machine moves its own.** For every machine — or the ones named — // the controller reads every row sealed to it, as the store holds it, and asks that machine's node-engine to move each // one, signed (link.AskMoveSecret): the engine opens it with its sealing key and hands it to the vault's move-in, sealed // to the call's key, keeping its origin and generation. The vault takes it from that engine alone, never over a value // it holds, and reads back what it kept to the engine, which compares; the engine answers only then. What moved is // recorded here, never a value. Nothing is rotated, and nothing points at the vault because of this: a module's // secrets are filled from the vault only once it moves (V4, issue 647). // // **A machine asleep is moved when it is next heard from**: once the move has started, a machine that reports is moved // in the background (moveWhenHeard), and what is left of it is said on the next `secret move`. // // The controller's record of asks at a desk, which holds no value, is copied as it is, once (`move-asks`). const secretMoveUsage = "secret move [] — move every secret the controller holds into the vault, as it is: each " + "machine's node-engine opens what was sealed to it and hands it to the vault, which reads it back; no value is " + "rotated, and a machine not answering is moved when it is next heard from" // moveAsk is the ask of one row of a machine's engine. type moveAsk func(row inventory.SecretToMove, ask link.MoveSecretAsk) (link.MoveSecretAnswer, error) // nodeMove is what the move did on one machine. type nodeMove struct { Node string Moved int Already int // Refused are the rows not moved, each with why; Changed the rows changed in the store after they moved. Refused []string Changed []string // Unreached is why the machine's engine could not be asked: it is moved when next heard from. Unreached error // Left is how many rows were not asked because the machine was not reached. Left int // Failed is why its rows could not be read. Failed error } // done says nothing is left to move on the machine. func (m nodeMove) done() bool { return m.Failed == nil && m.Unreached == nil && len(m.Refused) == 0 } func (m nodeMove) String() string { var b strings.Builder if m.Failed != nil { fmt.Fprintf(&b, "%s: its rows could not be read, nothing was moved: %v\n", m.Node, m.Failed) return b.String() } fmt.Fprintf(&b, "%s: %d moved into the vault and read back the same, %d there already", m.Node, m.Moved, m.Already) if n := len(m.Refused); n > 0 { fmt.Fprintf(&b, ", %d not moved", n) } b.WriteString("\n") for _, r := range m.Refused { fmt.Fprintf(&b, " not moved: %s\n", r) } for _, c := range m.Changed { fmt.Fprintf(&b, " %s was changed in the controller's store after it moved; the vault keeps what it took, "+ "and the move never replaces it\n", c) } if m.Unreached != nil { fmt.Fprintf(&b, " %d left: %s was not reached (%v); it is moved when it is next heard from\n", m.Left, m.Node, m.Unreached) } return b.String() } // moveRowAsk is a row as its machine's engine is asked to move it. func moveRowAsk(r inventory.SecretToMove) link.MoveSecretAsk { a := link.MoveSecretAsk{Origin: r.Origin, Generation: r.Generation, MadeAt: r.MadeAt.UTC(), Sealed: r.Sealed, SealedTo: r.SealedTo} if r.Own() { a.Module, a.Name = r.Module, r.Name } else { a.Provider, a.Provision, a.Consumer, a.ConsumerModule, a.Local = r.Provider, r.Provision, r.Consumer, r.ConsumerModule, r.Local } return a } // moveNode moves every row sealed to one machine that has not moved, one ask each, and records what the vault kept. // The first ask the engine does not answer ends the machine's turn: the rest wait for it to be heard from again. func moveNode(node string, rows func(string) ([]inventory.SecretToMove, error), ask moveAsk, record func(inventory.SecretToMove, int) error) nodeMove { m := nodeMove{Node: node} all, err := rows(node) if err != nil { m.Failed = err return m } for n, r := range all { switch { case r.Moved != nil && r.Moved.Digest == r.Digest: m.Already++ continue case r.Moved != nil: m.Changed = append(m.Changed, r.Secret) continue case !r.Current: m.Refused = append(m.Refused, fmt.Sprintf("%s is sealed to an earlier sealing key of %s, which no longer "+ "opens it: nothing can move it; it moves once it is made or accepted again", r.Secret, node)) continue } answer, err := ask(r, moveRowAsk(r)) if err != nil { m.Unreached, m.Left = err, len(all)-n return m } if answer.Refused != "" { m.Refused = append(m.Refused, fmt.Sprintf("%s: %s", r.Secret, answer.Refused)) continue } if answer.Secret != r.Secret { m.Refused = append(m.Refused, fmt.Sprintf("%s: the vault kept it as %s, not under the name the move "+ "gives it", r.Secret, answer.Secret)) continue } if err := record(r, answer.Generation); err != nil { m.Refused = append(m.Refused, fmt.Sprintf("%s is in the vault, but that it moved could not be recorded: %v", r.Secret, err)) continue } if answer.Unchanged { m.Already++ } else { m.Moved++ } } return m } // movedAsksAtOnce is how many rows of the record of asks one call to the vault carries (mesh-vault MovedAsksAtOnce). const movedAsksAtOnce = 500 // moveAsks copies the controller's record of asks into the vault, in parts, and answers how many it copied. func moveAsks(asks []inventory.SecretAsk, vault vaultAsker) (int, error) { copied := 0 for start := 0; start < len(asks); start += movedAsksAtOnce { end := min(start+movedAsksAtOnce, len(asks)) var rows []map[string]any for _, a := range asks[start:end] { row := map[string]any{"id": a.ID, "node": a.Node, "module": a.Module, "name": a.Name, "desk": a.Desk, "asked_by": a.AskedBy, "opened_at": a.OpenedAt.UTC().Format(time.RFC3339Nano)} if a.EndedAt != nil { row["ended_at"], row["outcome"] = a.EndedAt.UTC().Format(time.RFC3339Nano), a.Outcome } rows = append(rows, row) } raw, err := vault("move-asks", map[string]any{"asks": rows}) if err != nil { return copied, fmt.Errorf("the vault did not take the record of asks after %d rows: %w", copied, err) } var said struct { Copied int `json:"copied"` Already int `json:"already"` } _ = json.Unmarshal(raw, &said) copied += said.Copied + said.Already } return copied, nil } // moveOnTheBus is the move's ask over the bus, signed with the mesh's key. func moveOnTheBus(ctx context.Context, open *stores, conn *nats.Conn, by string) (moveAsk, error) { ident, err := open.Identity(ctx) if err != nil { return nil, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked: %w", err) } return func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { return link.AskMoveSecret(ctx, conn, ident, r.Node, by, row, link.MoveSecretWithin) }, nil } // secretMove is `secret move []`: the record of asks copied once, then every machine's rows — or one's — and // what moved said per machine. Any row not moved fails the command, said by name. func secretMove(ctx context.Context, args []string) error { if len(args) > 1 || (len(args) == 1 && (args[0] == "" || strings.HasPrefix(args[0], "-"))) { return errors.New(secretMoveUsage) } open, err := openStores(ctx) if err != nil { return err } defer open.Close() inv := open.inventory by := actingAs() if by == "" { by = handOverBy() } move, err := inv.StartSecretMove(ctx, by) if err != nil { return err } var nodes []string if len(args) == 1 { if _, err := inv.NodeByName(ctx, args[0]); err != nil { return fmt.Errorf("nothing was moved: %w", err) } nodes = args } else { all, err := inv.Nodes(ctx) if err != nil { return err } for _, n := range all { nodes = append(nodes, n.Name) } } fmt.Printf("the move of the secrets into the vault, started %s by %s\n", move.StartedAt.Local().Format("2006-01-02 15:04"), move.StartedBy) if move.AsksMovedAt == nil { asks, err := inv.SecretAsks(ctx, time.Time{}) if err != nil { return err } copied, err := moveAsks(asks, theVault(ctx)) if err != nil { return fmt.Errorf("the record of asks was not copied, and no secret was moved: %w", err) } if err := inv.RecordAsksMoved(ctx); err != nil { return err } fmt.Printf("the record of asks at a desk: %d rows copied into the vault as they are\n", copied) } failed := 0 err = onTheBus(func(conn *nats.Conn) error { ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+by) if err != nil { return err } for _, node := range nodes { m := moveNode(node, func(n string) ([]inventory.SecretToMove, error) { return inv.SecretsToMoveFrom(ctx, n) }, ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) }) fmt.Print(m.String()) if m.Failed != nil || len(m.Refused) > 0 { failed++ } } return nil }) if err != nil { return err } if failed > 0 { return fmt.Errorf("on %d machine(s) a secret was not moved, said above by name", failed) } return nil } // moveHeard is the machines being moved in the background now, and when each was last moved, so a machine that reports // often is not asked again before moveHeardEvery has passed. var moveHeard = struct { sync.Mutex running map[string]bool last map[string]time.Time }{running: map[string]bool{}, last: map[string]time.Time{}} // moveHeardEvery is how often a machine heard from is moved at most. const moveHeardEvery = 10 * time.Minute // moveWhenHeard moves what is left of a machine that was just heard from, once the move has started: a machine asleep // when it started is moved when it next reports. What it did is said on the controller's output, and recorded for the // next `secret move`. func moveWhenHeard(ctx context.Context, open *stores, conn *nats.Conn, node string) { if open == nil || conn == nil || node == "" { return } moveHeard.Lock() if moveHeard.running[node] || time.Since(moveHeard.last[node]) < moveHeardEvery { moveHeard.Unlock() return } moveHeard.running[node], moveHeard.last[node] = true, time.Now() moveHeard.Unlock() defer func() { moveHeard.Lock() delete(moveHeard.running, node) moveHeard.Unlock() }() inv := open.inventory move, started, err := inv.SecretMoveStarted(ctx) if err != nil || !started { return } rows, err := inv.SecretsToMoveFrom(ctx, node) if err != nil { return } left := false for _, r := range rows { left = left || (r.Moved == nil && r.Current) } if !left { return } ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+move.StartedBy+", on hearing from "+node) if err != nil { fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from and its secrets not moved: %v\n", node, err) return } m := moveNode(node, func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) }) if m.done() { fmt.Printf("mesh-controller: %s was heard from; the move of its secrets is done: %s", node, m.String()) return } fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from; the move of its secrets is not done: %s", node, m.String()) } // moveHeardOn is the connection a machine heard from is moved over: the serving controller's bus. func moveHeardOn() *nats.Conn { if js := servingBus.Load(); js != nil { return js.Conn() } return nil }