Replace a value given by hand like one the mesh made (hq ADR 0228)
A given own secret the module reads at start is held by nobody but that module, so the mesh need not read it to replace it: secret rotate now works on it, and a value given through secret accept is replaced on its own after the module's first good start under the mesh. Only a value an outside party issues (own-secrets "issued-by": "outside") or one the module applies stays as given, refused with the reason.
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// The serving controller hears every report; a clean one about the declaration a machine was last
|
||||
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
|
||||
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
|
||||
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
|
||||
// is read as the count of repairs a healer is wanted for.
|
||||
|
||||
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
|
||||
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
|
||||
// nor the new one, sealed to the machine as it was made.
|
||||
type SecretReplaced struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
Given time.Time `json:"given"`
|
||||
// Sent are the machines sent so the module starts again on the new value; Unsent says why
|
||||
// they could not be, in which case the next push carries it.
|
||||
Sent []string `json:"sent"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// givenEvents is where the serving controller states it; nil in a command.
|
||||
var givenEvents link.Bus
|
||||
|
||||
// replacing keeps one replacement per machine at a time: two reports arriving together find the
|
||||
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
|
||||
var replacing sync.Map
|
||||
|
||||
// startedWell says a report is a machine's clean account of a declaration: everything applied,
|
||||
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
|
||||
func startedWell(report link.Report) bool {
|
||||
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
|
||||
}
|
||||
|
||||
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
|
||||
|
||||
// replaceGiven replaces what the report makes due, sends the machines, and says so.
|
||||
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
|
||||
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
|
||||
return
|
||||
}
|
||||
defer replacing.Delete(report.Node)
|
||||
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
|
||||
if err != nil {
|
||||
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
|
||||
}
|
||||
for _, r := range replaced {
|
||||
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
|
||||
Sent: r.Machines, Why: givenWhy}
|
||||
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
|
||||
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
|
||||
if err := sendTo(ctx, open, r.Machines); err != nil {
|
||||
said.Sent, said.Unsent = nil, err.Error()
|
||||
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
|
||||
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
|
||||
}
|
||||
stateReplaced(ctx, said)
|
||||
}
|
||||
}
|
||||
|
||||
func stateReplaced(ctx context.Context, said SecretReplaced) {
|
||||
if givenEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(said)
|
||||
if err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
return
|
||||
}
|
||||
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
|
||||
// the machine is there is not (novox/hq ADR 0228).
|
||||
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
report link.Report
|
||||
good bool
|
||||
}{
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
|
||||
{link.Report{Node: "anchor"}, false},
|
||||
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
|
||||
} {
|
||||
if got := startedWell(c.report); got != c.good {
|
||||
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,7 +109,7 @@ func serve(ctx context.Context) error {
|
||||
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||
// something new is one thing that moves it, so the listener nudges it.
|
||||
statusFrom = newStatusSummary(composeStatus(open))
|
||||
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
|
||||
server, err := connectLink(ctx, inv, work, nudgingListener{Enrolment: work, summary: statusFrom, open: open})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -166,6 +166,8 @@ func serve(ctx context.Context) error {
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
||||
givenEvents = bus
|
||||
// Composed now and kept current, before the verb that answers from it is served.
|
||||
go statusFrom.keep(ctx)
|
||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||
|
||||
@@ -482,7 +482,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if err := need("node", "module", "secret"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
||||
}
|
||||
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||
argv := []string{"secret", "rotate", str("node"), str("module"), str("secret")}
|
||||
// Why, recorded in the hand-act log (novox/hq ADR 0228); a cause only beside a why.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||
// caller needs.
|
||||
|
||||
@@ -93,18 +93,31 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||
return nil
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
|
||||
untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||
// machine and the mesh cannot read it again.
|
||||
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
if untilStart {
|
||||
// Accepted, and said what it is for (novox/hq ADR 0228): a value given by hand adopts
|
||||
// something that already holds it, and lives only until the module has started on it.
|
||||
fmt.Printf(" it lives until %s next starts well under the mesh on it, and is then replaced with a value\n"+
|
||||
" the mesh makes, sealed and sent (ADR 0228): a value given by hand is for adopting something\n"+
|
||||
" already running that holds it\n", module)
|
||||
if record, err := open.inventory.NodeByName(ctx, node); err == nil && !record.Adopted {
|
||||
fmt.Printf(" %s is not an adopted machine: if %s is installed fresh there, it needs no given value —\n"+
|
||||
" the mesh makes one at the first push\n", node, module)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
}
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const secretUsage = "secret rotate <node> <module> <name>\n" +
|
||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
@@ -367,9 +380,20 @@ func valueFor(node, module, name, from string) (string, error) {
|
||||
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
|
||||
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
|
||||
// this is the secret with one party. Said in the log with who asked and when, never the value.
|
||||
//
|
||||
// **A value given to the mesh rotates the same way** (novox/hq ADR 0228): what a module reads at start
|
||||
// is held by nobody else, so the old value is not needed to replace it. Refused for a value an
|
||||
// outside party issued, which no value of the mesh's would replace. Why it was rotated is recorded in
|
||||
// the hand-act log when given — a rotation asked by a person is an act by hand, and a leak is a cause
|
||||
// worth counting.
|
||||
func secretRotate(ctx context.Context, args []string) error {
|
||||
rest, _ := split(args)
|
||||
if len(rest) != 3 {
|
||||
rest, flags := split(args)
|
||||
set := flag.NewFlagSet("secret rotate", flag.ContinueOnError)
|
||||
why := addHandActFlags(set)
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 3 || set.NArg() != 0 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
@@ -378,6 +402,10 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
origin, given, err := open.inventory.OwnSecretOrigin(ctx, node, module, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
|
||||
var refused inventory.ErrNotRotatable
|
||||
if errors.As(err, &refused) {
|
||||
@@ -385,8 +413,13 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "secret rotate", []string{node, module, name})
|
||||
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
|
||||
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
|
||||
if origin == inventory.OriginAccepted {
|
||||
fmt.Printf(" it replaces the value given to the mesh on %s: the mesh made this one, so `secret rotate` "+
|
||||
"replaces it again whenever asked (ADR 0228)\n", given.UTC().Format("2006-01-02"))
|
||||
}
|
||||
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
|
||||
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
|
||||
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
|
||||
|
||||
@@ -178,6 +178,9 @@ var readingVerbs = map[string]bool{
|
||||
type nudgingListener struct {
|
||||
link.Enrolment
|
||||
summary *statusSummary
|
||||
// open is the serving controller's stores, for replacing a given value after a module's first
|
||||
// good start (novox/hq ADR 0228).
|
||||
open *stores
|
||||
}
|
||||
|
||||
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||
@@ -189,5 +192,10 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
if err == nil && l.open != nil && startedWell(report) {
|
||||
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
||||
// nothing it caused (novox/hq ADR 0228).
|
||||
go replaceGiven(context.WithoutCancel(ctx), l.open, report)
|
||||
}
|
||||
return news, err
|
||||
}
|
||||
|
||||
@@ -24,6 +24,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
|
||||
states = append(states, conditions.HeartbeatEvent)
|
||||
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
states = append(states, link.KeySecretReplaced)
|
||||
for _, event := range states {
|
||||
if !slices.Contains(broker.ControllerStates, event) {
|
||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||
|
||||
@@ -208,7 +208,9 @@ var ControllerStates = []string{"applied", "refused", "built-before",
|
||||
"condition-raised", "condition-changed", "condition-cleared",
|
||||
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
||||
// machine that is not the control node, so the controller going quiet is itself said.
|
||||
"doctor-heartbeat"}
|
||||
"doctor-heartbeat",
|
||||
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced"}
|
||||
|
||||
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -168,7 +168,7 @@ func placedManifest(m Manifest, with Rendering) (Manifest, error) {
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
own[name] = OwnSecret{Path: filled, Taken: s.Taken}
|
||||
own[name] = OwnSecret{Path: filled, Taken: s.Taken, IssuedBy: s.IssuedBy}
|
||||
}
|
||||
m.OwnSecrets = own
|
||||
}
|
||||
|
||||
@@ -558,7 +558,10 @@ type Manifest struct {
|
||||
// that takes it only once, so a rotation must be staged beside the current value — the form the
|
||||
// mesh does not build yet, and refuses by name. A secret that says neither is not rotated by
|
||||
// the mesh: the one fault worse than an unrotated credential is a rotated one the software
|
||||
// never saw.
|
||||
// never saw. `"issued-by": "outside"` says a party outside the mesh issues the value — an API
|
||||
// key, a bot token, a licence — so the mesh never puts one of its own in its place (novox/hq
|
||||
// ADR 0228); any other at-start secret given by hand lives only until the module's first good
|
||||
// start, and is then replaced.
|
||||
OwnSecrets OwnSecrets `json:"own-secrets,omitempty"`
|
||||
|
||||
// SecretsOwner is who the files holding this module's secrets belong to on the machine —
|
||||
@@ -1801,6 +1804,13 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"or %q (applied by the module's own code to a backend that takes it once)",
|
||||
m.Module, name, own.Taken, TakenAtStart, TakenApplied))
|
||||
}
|
||||
if own.IssuedBy != "" && own.IssuedBy != IssuedOutside {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says its secret %q is issued by %q; a secret says %q when a party outside the mesh "+
|
||||
"issues it — a vendor's key, a bot's token, a licence — and says nothing when the mesh "+
|
||||
"may make it (novox/hq ADR 0228)",
|
||||
m.Module, name, own.IssuedBy, IssuedOutside))
|
||||
}
|
||||
}
|
||||
localOf := map[string]string{}
|
||||
for _, to := range m.SecretRequirements() {
|
||||
@@ -2263,10 +2273,24 @@ const (
|
||||
TakenApplied = "applied"
|
||||
)
|
||||
|
||||
// OwnSecret is where one of a module's own secrets lands, and how the module takes it.
|
||||
// IssuedOutside says a value was issued by a party outside the mesh — a vendor's API key, a bot's
|
||||
// token, a licence — so no value the mesh makes would work in its place (novox/hq ADR 0228).
|
||||
const IssuedOutside = "outside"
|
||||
|
||||
// OwnSecret is where one of a module's own secrets lands, how the module takes it, and — for a
|
||||
// value only an outside party can issue — that it is one.
|
||||
type OwnSecret struct {
|
||||
Path string
|
||||
Taken string
|
||||
Path string
|
||||
Taken string
|
||||
IssuedBy string
|
||||
}
|
||||
|
||||
// MeshMayMake says the mesh may put a value it makes in place of the one the secret holds: the
|
||||
// module reads it as it starts, and nobody outside the mesh issued it (novox/hq ADR 0228). A value
|
||||
// given to the mesh for such a secret lives only until the module's first good start under the
|
||||
// mesh; anything else given stays as given.
|
||||
func (s OwnSecret) MeshMayMake() bool {
|
||||
return s.Taken == TakenAtStart && s.IssuedBy != IssuedOutside
|
||||
}
|
||||
|
||||
// OwnSecrets is a module's own secrets by name. On the wire each is a path, or an object naming
|
||||
@@ -2287,15 +2311,16 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
|
||||
continue
|
||||
}
|
||||
var long struct {
|
||||
Path string `json:"path"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
Path string `json:"path"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
IssuedBy string `json:"issued-by,omitempty"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&long); err != nil {
|
||||
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\"}: %w", name, err)
|
||||
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, err)
|
||||
}
|
||||
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken}
|
||||
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy}
|
||||
}
|
||||
*o = out
|
||||
return nil
|
||||
@@ -2304,11 +2329,18 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
|
||||
func (o OwnSecrets) MarshalJSON() ([]byte, error) {
|
||||
entries := make(map[string]any, len(o))
|
||||
for name, s := range o {
|
||||
if s.Taken == "" {
|
||||
if s.Taken == "" && s.IssuedBy == "" {
|
||||
entries[name] = s.Path
|
||||
continue
|
||||
}
|
||||
entries[name] = map[string]string{"path": s.Path, "taken": s.Taken}
|
||||
long := map[string]string{"path": s.Path}
|
||||
if s.Taken != "" {
|
||||
long["taken"] = s.Taken
|
||||
}
|
||||
if s.IssuedBy != "" {
|
||||
long["issued-by"] = s.IssuedBy
|
||||
}
|
||||
entries[name] = long
|
||||
}
|
||||
return json.Marshal(entries)
|
||||
}
|
||||
|
||||
@@ -52,3 +52,42 @@ func TestAnOwnSecretTakenSomeOtherWayIsRefused(t *testing.T) {
|
||||
t.Fatal("an unknown field on an own secret was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A secret an outside party issues says so (novox/hq ADR 0228), is written back as it was read, and
|
||||
// is the one at-start secret the mesh may not make; any other word for who issued it is refused.
|
||||
func TestAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
|
||||
"server-password":{"path":"/var/lib/letta/server-password","taken":"at-start"},
|
||||
"openai-api-key":{"path":"/var/lib/letta/openai-api-key","taken":"at-start","issued-by":"outside"},
|
||||
"telegram-token":{"path":"/var/lib/letta/telegram-token","issued-by":"outside"},
|
||||
"logflare":{"path":"/var/lib/letta/logflare","taken":"applied"},
|
||||
"broker":"/var/lib/mesh/letta/broker"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, may := range map[string]bool{"server-password": true, "openai-api-key": false,
|
||||
"telegram-token": false, "logflare": false, "broker": false} {
|
||||
if got := m.OwnSecrets[name].MeshMayMake(); got != may {
|
||||
t.Errorf("%s: the mesh may make it = %v, want %v", name, got, may)
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m.OwnSecrets)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var again OwnSecrets
|
||||
if err := json.Unmarshal(out, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again["openai-api-key"] != m.OwnSecrets["openai-api-key"] || again["telegram-token"] != m.OwnSecrets["telegram-token"] {
|
||||
t.Fatalf("the round trip lost who issued it: %s", out)
|
||||
}
|
||||
if strings.Contains(string(out), `"taken":""`) {
|
||||
t.Fatalf("a secret that says only who issued it gained an empty taken: %s", out)
|
||||
}
|
||||
_, err = ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
|
||||
"openai-api-key":{"path":"/var/lib/letta/openai-api-key","issued-by":"openai"}}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), `issued by "openai"`) {
|
||||
t.Fatalf("an unknown word for who issued a secret was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +85,9 @@ var defaultSeats = append([]Seat{
|
||||
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
Emits: []string{"applied", "refused", "built-before",
|
||||
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat"},
|
||||
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat",
|
||||
// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced"},
|
||||
Serves: ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
|
||||
@@ -149,14 +149,17 @@ var ControllerVerbs = []Verb{
|
||||
}, []string{"why"}, "behind")},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
||||
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
||||
"name: made anew and the machine sent, so the module starts again on it — for a secret its definition " +
|
||||
"says it reads at start, whether the mesh made the value or a person gave it (novox/hq ADR 0228); one " +
|
||||
"an outside party issued, or one the module applies to a backend, is refused with the reason.",
|
||||
Input: schema(map[string]string{
|
||||
"provision": "a pair credential: the provision whose credential to replace",
|
||||
"consumer": "with provision: only the holder on this machine (optional)",
|
||||
"node": "an own secret: the machine",
|
||||
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
|
||||
"secret": "an own secret: its name in the module's definition",
|
||||
"why": "an own secret: why it is rotated — recorded in the hand-act log (optional)",
|
||||
"cause": "with why: the cause in a word, the word a second rotation for the same reason uses (optional)",
|
||||
}, nil)},
|
||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||
"machine as the module's own secret named broker, read at the next push of that machine. For a module " +
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// **A person gives a module's own secret for one reason**: to adopt something already running that
|
||||
// holds that value. A module the mesh installs fresh needs none — the mesh makes it. So for a secret
|
||||
// the mesh may make (catalogue.OwnSecret.MeshMayMake: read at start, issued by nobody outside), a
|
||||
// given value is kept until the module has started under the mesh on it, and then replaced with one
|
||||
// the mesh makes, sealed and sent like any other. Nothing a person handled is left in force.
|
||||
//
|
||||
// What stays as given: a value an outside party issued (an API key, a bot token, a licence), which
|
||||
// no value of the mesh's would replace; a value a module applies to a backend, until the staged
|
||||
// rotation exists (ADR 0114); and a value given before this rule, which waits for a person to ask
|
||||
// `secret rotate` — the mesh does not decide for them that what was given long ago is used nowhere
|
||||
// else.
|
||||
|
||||
// AcceptGivenSecret is `secret accept` for a module's own secret: the value a person gave, sealed as
|
||||
// AcceptSecretForModule seals it, and — for a secret the mesh may make — marked to be replaced after
|
||||
// the module's first good start. It answers whether it was so marked.
|
||||
//
|
||||
// **Only the person's path marks.** The mesh's own code accepts values too — a bus account it
|
||||
// issued, the controller's bus address — and those are the mesh's word to a broker, which a fresh
|
||||
// random value would break; they go through AcceptSecretForModule and are never marked.
|
||||
func (i *Inventory) AcceptGivenSecret(ctx context.Context, node, module, name, value string) (untilStart bool, err error) {
|
||||
return i.acceptOwn(ctx, node, module, name, value, true)
|
||||
}
|
||||
|
||||
// OwnSecretOrigin is where a module's own secret on a machine came from — OriginMade or
|
||||
// OriginAccepted — and when it was made or given; empty for one it does not hold yet.
|
||||
func (i *Inventory) OwnSecretOrigin(ctx context.Context, node, module, name string) (string, time.Time, error) {
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
var origin string
|
||||
var at time.Time
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select origin, made_at from module_secret where node = $1 and module = $2 and name = $3`,
|
||||
record.ID, module, name).Scan(&origin, &at)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", time.Time{}, nil
|
||||
}
|
||||
return origin, at, err
|
||||
}
|
||||
|
||||
// givenAgo is ", given <date>, N days ago" for a refusal about a value given to the mesh, and empty
|
||||
// when the mesh holds none: how old an outside party's key is, said where a person learns it cannot
|
||||
// be rotated by the mesh.
|
||||
func (i *Inventory) givenAgo(ctx context.Context, nodeID any, module, name string) string {
|
||||
var origin string
|
||||
var at time.Time
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select origin, made_at from module_secret where node = $1 and module = $2 and name = $3`,
|
||||
nodeID, module, name).Scan(&origin, &at)
|
||||
if err != nil || origin != OriginAccepted {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("; the value held was given %s, %d day(s) ago",
|
||||
at.UTC().Format("2006-01-02"), int(time.Since(at).Hours()/24))
|
||||
}
|
||||
|
||||
// GivenReplaced is one given value the mesh replaced after its module's first good start.
|
||||
type GivenReplaced struct {
|
||||
Module, Name string
|
||||
// Given is when the replaced value was given.
|
||||
Given time.Time
|
||||
// Machines are the machines to send so the module, and every holder of a shared credential,
|
||||
// starts again on the new value.
|
||||
Machines []string
|
||||
}
|
||||
|
||||
// ReplaceGivenAfterStart replaces every given value on a machine whose module has now started well
|
||||
// under the mesh on it (novox/hq ADR 0228), and answers what it replaced; the caller sends the
|
||||
// machines each names.
|
||||
//
|
||||
// **The signal is the machine's clean report of the declaration it was last sent** — every resource
|
||||
// applied, nothing failed or refused — **when that declaration was sent after the value was given**,
|
||||
// so it is the start on the given value that counts, not an older one. On an adopted machine the
|
||||
// module must also be taken: until then the mesh runs nothing of it, and nothing has started under
|
||||
// the mesh. Each row is claimed by clearing its mark before it is remade, so two reports arriving
|
||||
// together replace a value once; a remake that fails puts the mark back, and the next good report
|
||||
// tries again.
|
||||
func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared string) ([]GivenReplaced, error) {
|
||||
if declared == "" {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select s.node, s.module, s.name, s.replace_after_start
|
||||
from module_secret s
|
||||
join node n on n.id = s.node
|
||||
join assignment a on a.node = s.node and a.module = s.module
|
||||
where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
|
||||
and s.origin = 'accepted' and s.replace_after_start is not null
|
||||
and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
|
||||
order by s.module, s.name`, node, declared)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type due struct {
|
||||
nodeID any
|
||||
module, name string
|
||||
given time.Time
|
||||
}
|
||||
var dues []due
|
||||
for rows.Next() {
|
||||
var d due
|
||||
if err := rows.Scan(&d.nodeID, &d.module, &d.name, &d.given); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
dues = append(dues, d)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dues) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("%s has no sealing key, so the given values it holds cannot be replaced", node)
|
||||
}
|
||||
|
||||
var out []GivenReplaced
|
||||
var errs []error
|
||||
for _, d := range dues {
|
||||
claimed, err := i.store.Pool().Exec(ctx,
|
||||
`update module_secret set replace_after_start = null
|
||||
where node = $1 and module = $2 and name = $3 and origin = 'accepted'
|
||||
and replace_after_start = $4`, d.nodeID, d.module, d.name, d.given)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if claimed.RowsAffected() != 1 {
|
||||
continue // replaced by another report, or given again since
|
||||
}
|
||||
m, err := i.declared(ctx, d.module)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
// The definition is asked again now, not only when the value was given: one that has since
|
||||
// said the value is an outside party's, or applied, keeps it as given, and the mark stays gone.
|
||||
if own, ok := m.OwnSecrets[d.name]; !ok || !own.MeshMayMake() {
|
||||
continue
|
||||
}
|
||||
if err := i.remakeOwn(ctx, d.nodeID, key, m, d.module, d.name); err != nil {
|
||||
if _, back := i.store.Pool().Exec(ctx,
|
||||
`update module_secret set replace_after_start = $4
|
||||
where node = $1 and module = $2 and name = $3 and origin = 'accepted'
|
||||
and replace_after_start is null`, d.nodeID, d.module, d.name, d.given); back != nil {
|
||||
err = errors.Join(err, fmt.Errorf("and its mark could not be put back: %w", back))
|
||||
}
|
||||
errs = append(errs, fmt.Errorf("%s's given %q on %s was not replaced: %w", d.module, d.name, node, err))
|
||||
continue
|
||||
}
|
||||
machines, err := i.SharedHolders(ctx, node, d.module, d.name)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
machines = []string{node}
|
||||
}
|
||||
out = append(out, GivenReplaced{Module: d.module, Name: d.name, Given: d.given, Machines: machines})
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A module with one secret of each kind the rule tells apart (novox/hq ADR 0228), assigned to the
|
||||
// consumer machine.
|
||||
func aModuleWithGivenSecrets(t *testing.T) (*Inventory, context.Context) {
|
||||
t.Helper()
|
||||
inv, ctx := twoNodesWithKeys(t)
|
||||
m := catalogue.Manifest{Module: "letta", Version: "1", OwnSecrets: catalogue.OwnSecrets{
|
||||
"server-password": {Path: "/var/lib/letta/server-password", Taken: catalogue.TakenAtStart},
|
||||
"openai-api-key": {Path: "/var/lib/letta/openai-api-key", Taken: catalogue.TakenAtStart,
|
||||
IssuedBy: catalogue.IssuedOutside},
|
||||
"logflare": {Path: "/var/lib/letta/logflare", Taken: catalogue.TakenApplied},
|
||||
"broker": {Path: "/var/lib/mesh/letta/broker"},
|
||||
}}
|
||||
if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assign(t, inv, ctx, "consumer", "letta")
|
||||
return inv, ctx
|
||||
}
|
||||
|
||||
func assign(t *testing.T, inv *Inventory, ctx context.Context, node, module string) {
|
||||
t.Helper()
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.store.Pool().Exec(ctx,
|
||||
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`, n.ID, module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// sent records a declaration sent to a machine now, as a push does, and answers its digest.
|
||||
func sent(t *testing.T, inv *Inventory, ctx context.Context, node, digest string) string {
|
||||
t.Helper()
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, n.ID, digest, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
func originOf(t *testing.T, inv *Inventory, ctx context.Context, node, module, name string) string {
|
||||
t.Helper()
|
||||
origin, _, err := inv.OwnSecretOrigin(ctx, node, module, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return origin
|
||||
}
|
||||
|
||||
// **A given value is replaced once, after the first good start on it, and never again** (ADR 0228):
|
||||
// not on a report of a declaration sent before it was given, not on a report of a declaration the
|
||||
// mesh has moved past, and not a second time.
|
||||
func TestAGivenValueIsReplacedAfterTheFirstGoodStartAndNeverAgain(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
before := sent(t, inv, ctx, "consumer", "declaration-before")
|
||||
|
||||
marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed-by-a-person")
|
||||
if err != nil || !marked {
|
||||
t.Fatalf("a given value for a secret the mesh may make is marked to be replaced: %v %v", marked, err)
|
||||
}
|
||||
// The machine's report of what it was sent before the value was given is not a start on it.
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a start before the value was given replaced it: %+v %v", got, err)
|
||||
}
|
||||
carrying := sent(t, inv, ctx, "consumer", "declaration-carrying-it")
|
||||
// A report about a declaration other than the one last sent says nothing about this one.
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a report of an older declaration replaced it: %+v %v", got, err)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginAccepted {
|
||||
t.Fatal("the given value was replaced before its module started on it")
|
||||
}
|
||||
|
||||
got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Module != "letta" || got[0].Name != "server-password" ||
|
||||
len(got[0].Machines) != 1 || got[0].Machines[0] != "consumer" {
|
||||
t.Fatalf("the first good start replaced %+v", got)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginMade {
|
||||
t.Fatal("the replacement is not the mesh's own")
|
||||
}
|
||||
// Never again: the same report heard twice, and the next declaration's report.
|
||||
if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying); err != nil || len(again) != 0 {
|
||||
t.Fatalf("the same start replaced it twice: %+v %v", again, err)
|
||||
}
|
||||
next := sent(t, inv, ctx, "consumer", "declaration-after")
|
||||
if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", next); err != nil || len(again) != 0 {
|
||||
t.Fatalf("a later start replaced the mesh's own value: %+v %v", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **What stays as given** (ADR 0228): a value an outside party issued, a value the module applies,
|
||||
// and a value the mesh's own code accepted — a bus account it issued is the mesh's word to a broker,
|
||||
// and a fresh random value would break it.
|
||||
func TestWhatIsNeverReplacedAfterAStart(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
for _, name := range []string{"openai-api-key", "logflare"} {
|
||||
marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", name, "from-outside")
|
||||
if err != nil || marked {
|
||||
t.Fatalf("%s was marked to be replaced: %v %v", name, marked, err)
|
||||
}
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, "consumer", "letta", "broker", "issued-by-the-mesh"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := sent(t, inv, ctx, "consumer", "declaration")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a value that stays as given was replaced: %+v %v", got, err)
|
||||
}
|
||||
for _, name := range []string{"openai-api-key", "logflare", "broker"} {
|
||||
if originOf(t, inv, ctx, "consumer", "letta", name) != OriginAccepted {
|
||||
t.Fatalf("%s was touched", name)
|
||||
}
|
||||
}
|
||||
// And asked by hand, the outside party's key is refused, saying why and how old it is.
|
||||
var refused ErrNotRotatable
|
||||
err := inv.RotateModuleSecret(ctx, "consumer", "letta", "openai-api-key")
|
||||
if !errors.As(err, &refused) || !strings.Contains(err.Error(), "outside the mesh") ||
|
||||
!strings.Contains(err.Error(), "day(s) ago") || !strings.Contains(err.Error(), "secret accept") {
|
||||
t.Fatalf("rotating an outside party's key must be refused with its age and the way out: %v", err)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "openai-api-key") != OriginAccepted {
|
||||
t.Fatal("a refused rotation touched the value")
|
||||
}
|
||||
}
|
||||
|
||||
// On an adopted machine the module must be taken before a start is one under the mesh; and a module
|
||||
// no longer assigned to the machine has no start to wait for.
|
||||
func TestAGivenValueWaitsForTheTakeOnAnAdoptedMachine(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
if err := inv.SetAdopted(ctx, "consumer", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "the-found-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := sent(t, inv, ctx, "consumer", "declaration-holding-it")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", held); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a module held as found, not yet taken, had its given value replaced: %+v %v", got, err)
|
||||
}
|
||||
if err := inv.Take(ctx, "consumer", "letta"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taken := sent(t, inv, ctx, "consumer", "declaration-taking-it")
|
||||
got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", taken)
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("the first good start after the take did not replace the given value: %+v %v", got, err)
|
||||
}
|
||||
|
||||
// Unassigned: nothing starts, nothing is replaced.
|
||||
if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "given-again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, "consumer", "letta"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gone := sent(t, inv, ctx, "consumer", "declaration-without-it")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", gone); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a module no longer assigned had its given value replaced: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A definition that changed after the value was given is asked again at the start: one that now says
|
||||
// the value is an outside party's keeps it as given.
|
||||
func TestADefinitionThatNowSaysOutsideKeepsTheGivenValue(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := catalogue.Manifest{Module: "letta", Version: "2", OwnSecrets: catalogue.OwnSecrets{
|
||||
"server-password": {Path: "/var/lib/letta/server-password", Taken: catalogue.TakenAtStart,
|
||||
IssuedBy: catalogue.IssuedOutside}}}
|
||||
if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := sent(t, inv, ctx, "consumer", "declaration")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a value the definition now says is an outside party's was replaced: %+v %v", got, err)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginAccepted {
|
||||
t.Fatal("the value was touched")
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
-- A value given to the mesh for a secret it may make lives only until the module's first good start
|
||||
-- (novox/hq ADR 0228).
|
||||
--
|
||||
-- A person gives a module's own secret by hand for one reason: to adopt something already running
|
||||
-- that holds that value. Once the module has started under the mesh on it, the value has done its
|
||||
-- work, and the mesh puts one of its own in its place — made, sealed and sent as any value it makes,
|
||||
-- so nothing a person ever handled is left in force.
|
||||
--
|
||||
-- When the value was given, for a given value awaiting that replacement; null for every other row —
|
||||
-- a value the mesh made, one an outside party issued, one a module applies to a backend, and every
|
||||
-- value given before this column existed, which `secret rotate` replaces when a person asks. The
|
||||
-- replacement waits for a clean report of a declaration sent after this moment, so it is the start
|
||||
-- on the given value that is waited for, not an older one; and it clears the column, so it happens
|
||||
-- once.
|
||||
alter table module_secret add column replace_after_start timestamptz;
|
||||
@@ -373,7 +373,8 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
|
||||
on conflict (node, module, name) do update set
|
||||
sealed = excluded.sealed, node_key = excluded.node_key,
|
||||
origin = excluded.origin, made_at = now(),
|
||||
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
|
||||
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
|
||||
replace_after_start = null`,
|
||||
record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -390,49 +391,61 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
|
||||
// Sealed on the way in and the plaintext discarded, exactly as a generated one is — so the only
|
||||
// difference between the two is where the value came from.
|
||||
func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, name, value string) error {
|
||||
_, err := i.acceptOwn(ctx, node, module, name, value, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// acceptOwn is AcceptSecretForModule, and — with untilStart, for a value a person gave to a secret
|
||||
// the mesh may make — marks it to be replaced after the module's first good start (novox/hq ADR
|
||||
// 0228). It answers whether it was so marked.
|
||||
func (i *Inventory) acceptOwn(ctx context.Context, node, module, name, value string, untilStart bool) (bool, error) {
|
||||
// Refused for a name the module does not declare. A value stored under a name nothing reads
|
||||
// is a delivery that changed nothing and reported success — the shape of failure the mesh
|
||||
// is built to refuse (novox/hq 04-ISSUES/078).
|
||||
m, err := i.declared(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
if _, own := m.OwnSecrets[name]; !own {
|
||||
return fmt.Errorf("%s does not declare %q as an own secret; %s — a secret it requires from a provider is accepted with `--provider <node> [--local <name>]`, the value the running service already uses (novox/hq ADR 0163)", module, name, declaresOwn(m))
|
||||
own, declared := m.OwnSecrets[name]
|
||||
if !declared {
|
||||
return false, fmt.Errorf("%s does not declare %q as an own secret; %s — a secret it requires from a provider is accepted with `--provider <node> [--local <name>]`, the value the running service already uses (novox/hq ADR 0163)", module, name, declaresOwn(m))
|
||||
}
|
||||
untilStart = untilStart && own.MeshMayMake()
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
if key == "" {
|
||||
return fmt.Errorf(
|
||||
return false, fmt.Errorf(
|
||||
"%s has no sealing key, so nothing can be sealed to it — it joins again to get one",
|
||||
node)
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
|
||||
sealed, err := secrets.Accept(value, key, key)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
// And to the operator, when the mesh has one: a value a person supplied is the one a person
|
||||
// most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended).
|
||||
forOperator, operatorKey, err := i.operatorSeal(ctx, value)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
|
||||
values ($1, $2, $3, $4, $5, 'accepted', $6, $7)
|
||||
`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key,
|
||||
replace_after_start)
|
||||
values ($1, $2, $3, $4, $5, 'accepted', $6, $7, case when $8 then now() end)
|
||||
on conflict (node, module, name) do update set
|
||||
sealed = excluded.sealed, node_key = excluded.node_key,
|
||||
origin = excluded.origin, made_at = now(),
|
||||
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
|
||||
record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey)
|
||||
return err
|
||||
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
|
||||
replace_after_start = excluded.replace_after_start`,
|
||||
record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey, untilStart)
|
||||
return untilStart, err
|
||||
}
|
||||
|
||||
// Holder is one end-to-end credential: who gets it and who must create it.
|
||||
@@ -537,8 +550,12 @@ func (e ErrNotRotatable) Error() string { return e.Why }
|
||||
// backend that takes it once would, rotated this way, leave the backend on the old value and the
|
||||
// module reading the new one — the fault issue 179 was. That form is staged, which the mesh does
|
||||
// not build yet, and is refused by name. A secret whose manifest says neither is refused with the
|
||||
// word to write; a secret given to the mesh rather than made by it is refused as 0113 says: the
|
||||
// mesh will not replace what it cannot read.
|
||||
// word to write.
|
||||
//
|
||||
// **A value given to the mesh is replaced like one it made** (novox/hq ADR 0228, extending 0113):
|
||||
// the old value is not read, and need not be — a secret the module reads at start is held by
|
||||
// nobody but that module, so a fresh one sent to it is the whole change. Refused only for a value
|
||||
// an outside party issued (`"issued-by": "outside"`): no value the mesh makes would work there.
|
||||
func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name string) error {
|
||||
m, err := i.declared(ctx, module)
|
||||
if err != nil {
|
||||
@@ -548,6 +565,17 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
|
||||
if !declared {
|
||||
return fmt.Errorf("%s does not declare %q as an own secret; %s — a secret it requires from a provider is accepted with `--provider <node> [--local <name>]`, the value the running service already uses (novox/hq ADR 0163)", module, name, declaresOwn(m))
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if own.IssuedBy == catalogue.IssuedOutside {
|
||||
return ErrNotRotatable{Why: fmt.Sprintf(
|
||||
"%s's %q is issued by a party outside the mesh — its definition says \"issued-by\": "+
|
||||
"\"outside\" — so no value the mesh makes would work in its place (ADR 0228)%s. Have its "+
|
||||
"issuer make a new one, then `secret accept %s %s %s --from <file>`",
|
||||
module, name, i.givenAgo(ctx, record.ID, module, name), node, module, name)}
|
||||
}
|
||||
switch own.Taken {
|
||||
case catalogue.TakenAtStart:
|
||||
case catalogue.TakenApplied:
|
||||
@@ -564,10 +592,6 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
|
||||
"starts, or \"applied\" when its own code applies it",
|
||||
module, name, name)}
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -585,16 +609,16 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if origin == OriginAccepted {
|
||||
return ErrNotRotatable{Why: fmt.Sprintf(
|
||||
"%s on %s holds %q as a value given to the mesh, not made by it, and the mesh will not "+
|
||||
"replace what it cannot read (ADR 0113). Change it where it lives, then `secret accept "+
|
||||
"%s %s %s` with the new value",
|
||||
module, node, name, node, module, name)}
|
||||
}
|
||||
return i.remakeOwn(ctx, record.ID, key, m, module, name)
|
||||
}
|
||||
|
||||
// remakeOwn puts a value the mesh makes in a module's own secret, whatever it held: sealed to the
|
||||
// machine and the operator, origin made, and no longer awaiting a replacement. A secret that is a
|
||||
// provider's one credential (ADR 0158) is remade for every holder at once.
|
||||
func (i *Inventory) remakeOwn(ctx context.Context, nodeID any, key string, m catalogue.Manifest, module, name string) error {
|
||||
if len(m.ProvisionsSharing(name)) > 0 {
|
||||
// Shared with every consumer of those provisions (ADR 0158): one new value, sealed to all.
|
||||
return i.remakeShared(ctx, record.ID, key, module, name, "", nil, "", "", "")
|
||||
return i.remakeShared(ctx, nodeID, key, module, name, "", nil, "", "", "")
|
||||
}
|
||||
operator, err := i.OperatorKey(ctx)
|
||||
if err != nil {
|
||||
@@ -607,9 +631,9 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
|
||||
forOperator, operatorKey := operatorColumns(operator, blob)
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update module_secret set sealed = $4, node_key = $5, origin = 'made', made_at = now(),
|
||||
operator_sealed = $6, operator_key = $7
|
||||
operator_sealed = $6, operator_key = $7, replace_after_start = null
|
||||
where node = $1 and module = $2 and name = $3`,
|
||||
record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey)
|
||||
nodeID, module, name, made.ForConsumer, key, forOperator, operatorKey)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -725,7 +749,7 @@ func (i *Inventory) remakeShared(ctx context.Context, providerID any, providerKe
|
||||
on conflict (node, module, name) do update set
|
||||
sealed = excluded.sealed, node_key = excluded.node_key, origin = 'made', made_at = now(),
|
||||
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
|
||||
generation = excluded.generation`,
|
||||
generation = excluded.generation, replace_after_start = null`,
|
||||
providerID, providerModule, own, ownSealed, providerKey, forOperator, operatorKey, generation); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -789,9 +789,9 @@ func TestADeliveredPairCredentialIsRefusedForARequirementTheModuleDoesNotHave(t
|
||||
}
|
||||
|
||||
// A module's own secret rotates when its definition says the module reads it at start: made anew,
|
||||
// sealed to the machine and the operator, origin made. Refused with the reason when the definition
|
||||
// says nothing, says the module applies it, or when the value was given to the mesh (novox/hq
|
||||
// ADR 0114, issue 180).
|
||||
// sealed to the machine and the operator, origin made — whether the mesh made the value or a person
|
||||
// gave it (novox/hq ADR 0228). Refused with the reason when the definition says nothing or says the
|
||||
// module applies it (novox/hq ADR 0114, issue 180).
|
||||
func TestAnOwnSecretRotatesOnlyWhenTheModuleReadsItAtStart(t *testing.T) {
|
||||
inv, ctx := twoNodesWithKeys(t)
|
||||
m := catalogue.Manifest{Module: "idp", Version: "1", OwnSecrets: catalogue.OwnSecrets{
|
||||
@@ -833,12 +833,23 @@ func TestAnOwnSecretRotatesOnlyWhenTheModuleReadsItAtStart(t *testing.T) {
|
||||
t.Fatalf("a secret that says nothing of how it is taken must be refused: %v", err)
|
||||
}
|
||||
|
||||
// A value given to the mesh for a secret read at start is replaced like one it made (ADR 0228).
|
||||
if err := inv.AcceptSecretForModule(ctx, "consumer", "idp", "session", "the-real-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = inv.RotateModuleSecret(ctx, "consumer", "idp", "session")
|
||||
if !errors.As(err, &refused) || !strings.Contains(err.Error(), "given to the mesh") {
|
||||
t.Fatalf("an accepted value must be refused: %v", err)
|
||||
if err := inv.RotateModuleSecret(ctx, "consumer", "idp", "session"); err != nil {
|
||||
t.Fatalf("a given value read at start must rotate: %v", err)
|
||||
}
|
||||
if origin, _, err := inv.OwnSecretOrigin(ctx, "consumer", "idp", "session"); err != nil || origin != OriginMade {
|
||||
t.Fatalf("a rotated given value is the mesh's own from then on: %q %v", origin, err)
|
||||
}
|
||||
// A given value the module applies stays refused, with the reason.
|
||||
if err := inv.AcceptSecretForModule(ctx, "consumer", "idp", "admin", "the-real-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = inv.RotateModuleSecret(ctx, "consumer", "idp", "admin")
|
||||
if !errors.As(err, &refused) || !strings.Contains(err.Error(), "staged") {
|
||||
t.Fatalf("a given value the module applies must be refused: %v", err)
|
||||
}
|
||||
if err := inv.RotateModuleSecret(ctx, "consumer", "idp", "nothing"); err == nil || !strings.Contains(err.Error(), "does not declare") {
|
||||
t.Fatalf("an undeclared secret: %v", err)
|
||||
|
||||
@@ -63,6 +63,9 @@ const (
|
||||
// KeyBuiltBefore: a build the mesh already held, for a catalogue that asked what it missed. Not
|
||||
// `built` — that is the build machine's, said as it happens, and a replay is neither.
|
||||
KeyBuiltBefore = "built-before"
|
||||
// KeySecretReplaced: a value given to the mesh by hand was replaced with one it made, after its
|
||||
// module's first good start (novox/hq ADR 0228). Never the value.
|
||||
KeySecretReplaced = "secret-replaced"
|
||||
)
|
||||
|
||||
// Applied is what a machine now runs, as the mesh states it.
|
||||
|
||||
Reference in New Issue
Block a user