Replace a value given by hand like one the mesh made (hq ADR 0228)

A given own secret the module reads at start is held by nobody but that
module, so the mesh need not read it to replace it: secret rotate now
works on it, and a value given through secret accept is replaced on its
own after the module's first good start under the mesh. Only a value an
outside party issues (own-secrets "issued-by": "outside") or one the
module applies stays as given, refused with the reason.

internal/broker/states_agreement_test.go

@@ -24,6 +24,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
 	// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
 	states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
 	states = append(states, conditions.HeartbeatEvent)
+	// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
+	states = append(states, link.KeySecretReplaced)
 	for _, event := range states {
 		if !slices.Contains(broker.ControllerStates, event) {
 			t.Errorf("the mesh states %q and its account may not publish it", event)

internal/broker/streams.go

@@ -208,7 +208,9 @@ var ControllerStates = []string{"applied", "refused", "built-before",
 	"condition-raised", "condition-changed", "condition-cleared",
 	// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
 	// machine that is not the control node, so the controller going quiet is itself said.
-	"doctor-heartbeat"}
+	"doctor-heartbeat",
+	// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
+	"secret-replaced"}
 
 // BusAdvisories are what the bus server says about the mesh's own account that the controller
 // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it

internal/broker/testdata/composed.conf

@@ -24,7 +24,7 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/catalogue/dir_into.go

@@ -168,7 +168,7 @@ func placedManifest(m Manifest, with Rendering) (Manifest, error) {
 			if err != nil {
 				return m, err
 			}
-			own[name] = OwnSecret{Path: filled, Taken: s.Taken}
+			own[name] = OwnSecret{Path: filled, Taken: s.Taken, IssuedBy: s.IssuedBy}
 		}
 		m.OwnSecrets = own
 	}

internal/catalogue/manifest.go

@@ -558,7 +558,10 @@ type Manifest struct {
 	// that takes it only once, so a rotation must be staged beside the current value — the form the
 	// mesh does not build yet, and refuses by name. A secret that says neither is not rotated by
 	// the mesh: the one fault worse than an unrotated credential is a rotated one the software
-	// never saw.
+	// never saw. `"issued-by": "outside"` says a party outside the mesh issues the value — an API
+	// key, a bot token, a licence — so the mesh never puts one of its own in its place (novox/hq
+	// ADR 0228); any other at-start secret given by hand lives only until the module's first good
+	// start, and is then replaced.
 	OwnSecrets OwnSecrets `json:"own-secrets,omitempty"`
 
 	// SecretsOwner is who the files holding this module's secrets belong to on the machine —
@@ -1801,6 +1804,13 @@ func ParseManifest(raw []byte) (Manifest, error) {
 					"or %q (applied by the module's own code to a backend that takes it once)",
 				m.Module, name, own.Taken, TakenAtStart, TakenApplied))
 		}
+		if own.IssuedBy != "" && own.IssuedBy != IssuedOutside {
+			problems = append(problems, fmt.Sprintf(
+				"%s says its secret %q is issued by %q; a secret says %q when a party outside the mesh "+
+					"issues it — a vendor's key, a bot's token, a licence — and says nothing when the mesh "+
+					"may make it (novox/hq ADR 0228)",
+				m.Module, name, own.IssuedBy, IssuedOutside))
+		}
 	}
 	localOf := map[string]string{}
 	for _, to := range m.SecretRequirements() {
@@ -2263,10 +2273,24 @@ const (
 	TakenApplied = "applied"
 )
 
-// OwnSecret is where one of a module's own secrets lands, and how the module takes it.
+// IssuedOutside says a value was issued by a party outside the mesh — a vendor's API key, a bot's
+// token, a licence — so no value the mesh makes would work in its place (novox/hq ADR 0228).
+const IssuedOutside = "outside"
+
+// OwnSecret is where one of a module's own secrets lands, how the module takes it, and — for a
+// value only an outside party can issue — that it is one.
 type OwnSecret struct {
-	Path  string
-	Taken string
+	Path     string
+	Taken    string
+	IssuedBy string
+}
+
+// MeshMayMake says the mesh may put a value it makes in place of the one the secret holds: the
+// module reads it as it starts, and nobody outside the mesh issued it (novox/hq ADR 0228). A value
+// given to the mesh for such a secret lives only until the module's first good start under the
+// mesh; anything else given stays as given.
+func (s OwnSecret) MeshMayMake() bool {
+	return s.Taken == TakenAtStart && s.IssuedBy != IssuedOutside
 }
 
 // OwnSecrets is a module's own secrets by name. On the wire each is a path, or an object naming
@@ -2287,15 +2311,16 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
 			continue
 		}
 		var long struct {
-			Path  string `json:"path"`
-			Taken string `json:"taken,omitempty"`
+			Path     string `json:"path"`
+			Taken    string `json:"taken,omitempty"`
+			IssuedBy string `json:"issued-by,omitempty"`
 		}
 		dec := json.NewDecoder(bytes.NewReader(body))
 		dec.DisallowUnknownFields()
 		if err := dec.Decode(&long); err != nil {
-			return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\"}: %w", name, err)
+			return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, err)
 		}
-		out[name] = OwnSecret{Path: long.Path, Taken: long.Taken}
+		out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy}
 	}
 	*o = out
 	return nil
@@ -2304,11 +2329,18 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
 func (o OwnSecrets) MarshalJSON() ([]byte, error) {
 	entries := make(map[string]any, len(o))
 	for name, s := range o {
-		if s.Taken == "" {
+		if s.Taken == "" && s.IssuedBy == "" {
 			entries[name] = s.Path
 			continue
 		}
-		entries[name] = map[string]string{"path": s.Path, "taken": s.Taken}
+		long := map[string]string{"path": s.Path}
+		if s.Taken != "" {
+			long["taken"] = s.Taken
+		}
+		if s.IssuedBy != "" {
+			long["issued-by"] = s.IssuedBy
+		}
+		entries[name] = long
 	}
 	return json.Marshal(entries)
 }

internal/catalogue/own_secret_taken_test.go

@@ -52,3 +52,42 @@ func TestAnOwnSecretTakenSomeOtherWayIsRefused(t *testing.T) {
 		t.Fatal("an unknown field on an own secret was accepted")
 	}
 }
+
+// A secret an outside party issues says so (novox/hq ADR 0228), is written back as it was read, and
+// is the one at-start secret the mesh may not make; any other word for who issued it is refused.
+func TestAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) {
+	m, err := ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
+		"server-password":{"path":"/var/lib/letta/server-password","taken":"at-start"},
+		"openai-api-key":{"path":"/var/lib/letta/openai-api-key","taken":"at-start","issued-by":"outside"},
+		"telegram-token":{"path":"/var/lib/letta/telegram-token","issued-by":"outside"},
+		"logflare":{"path":"/var/lib/letta/logflare","taken":"applied"},
+		"broker":"/var/lib/mesh/letta/broker"}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, may := range map[string]bool{"server-password": true, "openai-api-key": false,
+		"telegram-token": false, "logflare": false, "broker": false} {
+		if got := m.OwnSecrets[name].MeshMayMake(); got != may {
+			t.Errorf("%s: the mesh may make it = %v, want %v", name, got, may)
+		}
+	}
+	out, err := json.Marshal(m.OwnSecrets)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var again OwnSecrets
+	if err := json.Unmarshal(out, &again); err != nil {
+		t.Fatal(err)
+	}
+	if again["openai-api-key"] != m.OwnSecrets["openai-api-key"] || again["telegram-token"] != m.OwnSecrets["telegram-token"] {
+		t.Fatalf("the round trip lost who issued it: %s", out)
+	}
+	if strings.Contains(string(out), `"taken":""`) {
+		t.Fatalf("a secret that says only who issued it gained an empty taken: %s", out)
+	}
+	_, err = ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
+		"openai-api-key":{"path":"/var/lib/letta/openai-api-key","issued-by":"openai"}}}`))
+	if err == nil || !strings.Contains(err.Error(), `issued by "openai"`) {
+		t.Fatalf("an unknown word for who issued a secret was accepted: %v", err)
+	}
+}

internal/catalogue/seats.go

@@ -85,7 +85,9 @@ var defaultSeats = append([]Seat{
 	{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
 		// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
 		Emits: []string{"applied", "refused", "built-before",
-			"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat"},
+			"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat",
+			// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
+			"secret-replaced"},
 		Serves: ControllerVerbs},
 	// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
 	// served by whichever module holds the seat with tools of these names.

internal/catalogue/verbs.go

@@ -149,14 +149,17 @@ var ControllerVerbs = []Verb{
 		}, []string{"why"}, "behind")},
 	{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
 		"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
-		"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
-		"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
+		"name: made anew and the machine sent, so the module starts again on it — for a secret its definition " +
+		"says it reads at start, whether the mesh made the value or a person gave it (novox/hq ADR 0228); one " +
+		"an outside party issued, or one the module applies to a backend, is refused with the reason.",
 		Input: schema(map[string]string{
 			"provision": "a pair credential: the provision whose credential to replace",
 			"consumer":  "with provision: only the holder on this machine (optional)",
 			"node":      "an own secret: the machine",
 			"module":    "an own secret: the module; with provision: only this consuming module's credential (optional)",
 			"secret":    "an own secret: its name in the module's definition",
+			"why":       "an own secret: why it is rotated — recorded in the hand-act log (optional)",
+			"cause":     "with why: the cause in a word, the word a second rotation for the same reason uses (optional)",
 		}, nil)},
 	{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
 		"machine as the module's own secret named broker, read at the next push of that machine. For a module " +

internal/inventory/given.go

@@ -0,0 +1,181 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
+//
+// **A person gives a module's own secret for one reason**: to adopt something already running that
+// holds that value. A module the mesh installs fresh needs none — the mesh makes it. So for a secret
+// the mesh may make (catalogue.OwnSecret.MeshMayMake: read at start, issued by nobody outside), a
+// given value is kept until the module has started under the mesh on it, and then replaced with one
+// the mesh makes, sealed and sent like any other. Nothing a person handled is left in force.
+//
+// What stays as given: a value an outside party issued (an API key, a bot token, a licence), which
+// no value of the mesh's would replace; a value a module applies to a backend, until the staged
+// rotation exists (ADR 0114); and a value given before this rule, which waits for a person to ask
+// `secret rotate` — the mesh does not decide for them that what was given long ago is used nowhere
+// else.
+
+// AcceptGivenSecret is `secret accept` for a module's own secret: the value a person gave, sealed as
+// AcceptSecretForModule seals it, and — for a secret the mesh may make — marked to be replaced after
+// the module's first good start. It answers whether it was so marked.
+//
+// **Only the person's path marks.** The mesh's own code accepts values too — a bus account it
+// issued, the controller's bus address — and those are the mesh's word to a broker, which a fresh
+// random value would break; they go through AcceptSecretForModule and are never marked.
+func (i *Inventory) AcceptGivenSecret(ctx context.Context, node, module, name, value string) (untilStart bool, err error) {
+	return i.acceptOwn(ctx, node, module, name, value, true)
+}
+
+// OwnSecretOrigin is where a module's own secret on a machine came from — OriginMade or
+// OriginAccepted — and when it was made or given; empty for one it does not hold yet.
+func (i *Inventory) OwnSecretOrigin(ctx context.Context, node, module, name string) (string, time.Time, error) {
+	record, err := i.NodeByName(ctx, node)
+	if err != nil {
+		return "", time.Time{}, err
+	}
+	var origin string
+	var at time.Time
+	err = i.store.Pool().QueryRow(ctx,
+		`select origin, made_at from module_secret where node = $1 and module = $2 and name = $3`,
+		record.ID, module, name).Scan(&origin, &at)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return "", time.Time{}, nil
+	}
+	return origin, at, err
+}
+
+// givenAgo is ", given <date>, N days ago" for a refusal about a value given to the mesh, and empty
+// when the mesh holds none: how old an outside party's key is, said where a person learns it cannot
+// be rotated by the mesh.
+func (i *Inventory) givenAgo(ctx context.Context, nodeID any, module, name string) string {
+	var origin string
+	var at time.Time
+	err := i.store.Pool().QueryRow(ctx,
+		`select origin, made_at from module_secret where node = $1 and module = $2 and name = $3`,
+		nodeID, module, name).Scan(&origin, &at)
+	if err != nil || origin != OriginAccepted {
+		return ""
+	}
+	return fmt.Sprintf("; the value held was given %s, %d day(s) ago",
+		at.UTC().Format("2006-01-02"), int(time.Since(at).Hours()/24))
+}
+
+// GivenReplaced is one given value the mesh replaced after its module's first good start.
+type GivenReplaced struct {
+	Module, Name string
+	// Given is when the replaced value was given.
+	Given time.Time
+	// Machines are the machines to send so the module, and every holder of a shared credential,
+	// starts again on the new value.
+	Machines []string
+}
+
+// ReplaceGivenAfterStart replaces every given value on a machine whose module has now started well
+// under the mesh on it (novox/hq ADR 0228), and answers what it replaced; the caller sends the
+// machines each names.
+//
+// **The signal is the machine's clean report of the declaration it was last sent** — every resource
+// applied, nothing failed or refused — **when that declaration was sent after the value was given**,
+// so it is the start on the given value that counts, not an older one. On an adopted machine the
+// module must also be taken: until then the mesh runs nothing of it, and nothing has started under
+// the mesh. Each row is claimed by clearing its mark before it is remade, so two reports arriving
+// together replace a value once; a remake that fails puts the mark back, and the next good report
+// tries again.
+func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared string) ([]GivenReplaced, error) {
+	if declared == "" {
+		return nil, nil
+	}
+	rows, err := i.store.Pool().Query(ctx,
+		`select s.node, s.module, s.name, s.replace_after_start
+		   from module_secret s
+		   join node n on n.id = s.node
+		   join assignment a on a.node = s.node and a.module = s.module
+		  where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
+		    and s.origin = 'accepted' and s.replace_after_start is not null
+		    and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
+		  order by s.module, s.name`, node, declared)
+	if err != nil {
+		return nil, err
+	}
+	type due struct {
+		nodeID       any
+		module, name string
+		given        time.Time
+	}
+	var dues []due
+	for rows.Next() {
+		var d due
+		if err := rows.Scan(&d.nodeID, &d.module, &d.name, &d.given); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		dues = append(dues, d)
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	if len(dues) == 0 {
+		return nil, nil
+	}
+	key, err := i.SealingKeyOf(ctx, node)
+	if err != nil {
+		return nil, err
+	}
+	if key == "" {
+		return nil, fmt.Errorf("%s has no sealing key, so the given values it holds cannot be replaced", node)
+	}
+
+	var out []GivenReplaced
+	var errs []error
+	for _, d := range dues {
+		claimed, err := i.store.Pool().Exec(ctx,
+			`update module_secret set replace_after_start = null
+			  where node = $1 and module = $2 and name = $3 and origin = 'accepted'
+			    and replace_after_start = $4`, d.nodeID, d.module, d.name, d.given)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if claimed.RowsAffected() != 1 {
+			continue // replaced by another report, or given again since
+		}
+		m, err := i.declared(ctx, d.module)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		// The definition is asked again now, not only when the value was given: one that has since
+		// said the value is an outside party's, or applied, keeps it as given, and the mark stays gone.
+		if own, ok := m.OwnSecrets[d.name]; !ok || !own.MeshMayMake() {
+			continue
+		}
+		if err := i.remakeOwn(ctx, d.nodeID, key, m, d.module, d.name); err != nil {
+			if _, back := i.store.Pool().Exec(ctx,
+				`update module_secret set replace_after_start = $4
+				  where node = $1 and module = $2 and name = $3 and origin = 'accepted'
+				    and replace_after_start is null`, d.nodeID, d.module, d.name, d.given); back != nil {
+				err = errors.Join(err, fmt.Errorf("and its mark could not be put back: %w", back))
+			}
+			errs = append(errs, fmt.Errorf("%s's given %q on %s was not replaced: %w", d.module, d.name, node, err))
+			continue
+		}
+		machines, err := i.SharedHolders(ctx, node, d.module, d.name)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		if len(machines) == 0 {
+			machines = []string{node}
+		}
+		out = append(out, GivenReplaced{Module: d.module, Name: d.name, Given: d.given, Machines: machines})
+	}
+	return out, errors.Join(errs...)
+}

internal/inventory/given_test.go

@@ -0,0 +1,201 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// A module with one secret of each kind the rule tells apart (novox/hq ADR 0228), assigned to the
+// consumer machine.
+func aModuleWithGivenSecrets(t *testing.T) (*Inventory, context.Context) {
+	t.Helper()
+	inv, ctx := twoNodesWithKeys(t)
+	m := catalogue.Manifest{Module: "letta", Version: "1", OwnSecrets: catalogue.OwnSecrets{
+		"server-password": {Path: "/var/lib/letta/server-password", Taken: catalogue.TakenAtStart},
+		"openai-api-key": {Path: "/var/lib/letta/openai-api-key", Taken: catalogue.TakenAtStart,
+			IssuedBy: catalogue.IssuedOutside},
+		"logflare": {Path: "/var/lib/letta/logflare", Taken: catalogue.TakenApplied},
+		"broker":   {Path: "/var/lib/mesh/letta/broker"},
+	}}
+	if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
+		t.Fatal(err)
+	}
+	assign(t, inv, ctx, "consumer", "letta")
+	return inv, ctx
+}
+
+func assign(t *testing.T, inv *Inventory, ctx context.Context, node, module string) {
+	t.Helper()
+	n, err := inv.NodeByName(ctx, node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.store.Pool().Exec(ctx,
+		`insert into assignment (node, module) values ($1, $2) on conflict do nothing`, n.ID, module); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// sent records a declaration sent to a machine now, as a push does, and answers its digest.
+func sent(t *testing.T, inv *Inventory, ctx context.Context, node, digest string) string {
+	t.Helper()
+	n, err := inv.NodeByName(ctx, node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, n.ID, digest, nil); err != nil {
+		t.Fatal(err)
+	}
+	return digest
+}
+
+func originOf(t *testing.T, inv *Inventory, ctx context.Context, node, module, name string) string {
+	t.Helper()
+	origin, _, err := inv.OwnSecretOrigin(ctx, node, module, name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return origin
+}
+
+// **A given value is replaced once, after the first good start on it, and never again** (ADR 0228):
+// not on a report of a declaration sent before it was given, not on a report of a declaration the
+// mesh has moved past, and not a second time.
+func TestAGivenValueIsReplacedAfterTheFirstGoodStartAndNeverAgain(t *testing.T) {
+	inv, ctx := aModuleWithGivenSecrets(t)
+	before := sent(t, inv, ctx, "consumer", "declaration-before")
+
+	marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed-by-a-person")
+	if err != nil || !marked {
+		t.Fatalf("a given value for a secret the mesh may make is marked to be replaced: %v %v", marked, err)
+	}
+	// The machine's report of what it was sent before the value was given is not a start on it.
+	if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 {
+		t.Fatalf("a start before the value was given replaced it: %+v %v", got, err)
+	}
+	carrying := sent(t, inv, ctx, "consumer", "declaration-carrying-it")
+	// A report about a declaration other than the one last sent says nothing about this one.
+	if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 {
+		t.Fatalf("a report of an older declaration replaced it: %+v %v", got, err)
+	}
+	if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginAccepted {
+		t.Fatal("the given value was replaced before its module started on it")
+	}
+
+	got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Module != "letta" || got[0].Name != "server-password" ||
+		len(got[0].Machines) != 1 || got[0].Machines[0] != "consumer" {
+		t.Fatalf("the first good start replaced %+v", got)
+	}
+	if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginMade {
+		t.Fatal("the replacement is not the mesh's own")
+	}
+	// Never again: the same report heard twice, and the next declaration's report.
+	if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying); err != nil || len(again) != 0 {
+		t.Fatalf("the same start replaced it twice: %+v %v", again, err)
+	}
+	next := sent(t, inv, ctx, "consumer", "declaration-after")
+	if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", next); err != nil || len(again) != 0 {
+		t.Fatalf("a later start replaced the mesh's own value: %+v %v", again, err)
+	}
+}
+
+// **What stays as given** (ADR 0228): a value an outside party issued, a value the module applies,
+// and a value the mesh's own code accepted — a bus account it issued is the mesh's word to a broker,
+// and a fresh random value would break it.
+func TestWhatIsNeverReplacedAfterAStart(t *testing.T) {
+	inv, ctx := aModuleWithGivenSecrets(t)
+	for _, name := range []string{"openai-api-key", "logflare"} {
+		marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", name, "from-outside")
+		if err != nil || marked {
+			t.Fatalf("%s was marked to be replaced: %v %v", name, marked, err)
+		}
+	}
+	if err := inv.AcceptSecretForModule(ctx, "consumer", "letta", "broker", "issued-by-the-mesh"); err != nil {
+		t.Fatal(err)
+	}
+	declared := sent(t, inv, ctx, "consumer", "declaration")
+	if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 {
+		t.Fatalf("a value that stays as given was replaced: %+v %v", got, err)
+	}
+	for _, name := range []string{"openai-api-key", "logflare", "broker"} {
+		if originOf(t, inv, ctx, "consumer", "letta", name) != OriginAccepted {
+			t.Fatalf("%s was touched", name)
+		}
+	}
+	// And asked by hand, the outside party's key is refused, saying why and how old it is.
+	var refused ErrNotRotatable
+	err := inv.RotateModuleSecret(ctx, "consumer", "letta", "openai-api-key")
+	if !errors.As(err, &refused) || !strings.Contains(err.Error(), "outside the mesh") ||
+		!strings.Contains(err.Error(), "day(s) ago") || !strings.Contains(err.Error(), "secret accept") {
+		t.Fatalf("rotating an outside party's key must be refused with its age and the way out: %v", err)
+	}
+	if originOf(t, inv, ctx, "consumer", "letta", "openai-api-key") != OriginAccepted {
+		t.Fatal("a refused rotation touched the value")
+	}
+}
+
+// On an adopted machine the module must be taken before a start is one under the mesh; and a module
+// no longer assigned to the machine has no start to wait for.
+func TestAGivenValueWaitsForTheTakeOnAnAdoptedMachine(t *testing.T) {
+	inv, ctx := aModuleWithGivenSecrets(t)
+	if err := inv.SetAdopted(ctx, "consumer", true); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "the-found-one"); err != nil {
+		t.Fatal(err)
+	}
+	held := sent(t, inv, ctx, "consumer", "declaration-holding-it")
+	if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", held); err != nil || len(got) != 0 {
+		t.Fatalf("a module held as found, not yet taken, had its given value replaced: %+v %v", got, err)
+	}
+	if err := inv.Take(ctx, "consumer", "letta"); err != nil {
+		t.Fatal(err)
+	}
+	taken := sent(t, inv, ctx, "consumer", "declaration-taking-it")
+	got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", taken)
+	if err != nil || len(got) != 1 {
+		t.Fatalf("the first good start after the take did not replace the given value: %+v %v", got, err)
+	}
+
+	// Unassigned: nothing starts, nothing is replaced.
+	if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "given-again"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.Unassign(ctx, "consumer", "letta"); err != nil {
+		t.Fatal(err)
+	}
+	gone := sent(t, inv, ctx, "consumer", "declaration-without-it")
+	if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", gone); err != nil || len(got) != 0 {
+		t.Fatalf("a module no longer assigned had its given value replaced: %+v %v", got, err)
+	}
+}
+
+// A definition that changed after the value was given is asked again at the start: one that now says
+// the value is an outside party's keeps it as given.
+func TestADefinitionThatNowSaysOutsideKeepsTheGivenValue(t *testing.T) {
+	inv, ctx := aModuleWithGivenSecrets(t)
+	if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed"); err != nil {
+		t.Fatal(err)
+	}
+	m := catalogue.Manifest{Module: "letta", Version: "2", OwnSecrets: catalogue.OwnSecrets{
+		"server-password": {Path: "/var/lib/letta/server-password", Taken: catalogue.TakenAtStart,
+			IssuedBy: catalogue.IssuedOutside}}}
+	if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
+		t.Fatal(err)
+	}
+	declared := sent(t, inv, ctx, "consumer", "declaration")
+	if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 {
+		t.Fatalf("a value the definition now says is an outside party's was replaced: %+v %v", got, err)
+	}
+	if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginAccepted {
+		t.Fatal("the value was touched")
+	}
+}

internal/inventory/migrations/0067-a-given-secret-lives-until-the-first-good-start.sql

@@ -0,0 +1,15 @@
+-- A value given to the mesh for a secret it may make lives only until the module's first good start
+-- (novox/hq ADR 0228).
+--
+-- A person gives a module's own secret by hand for one reason: to adopt something already running
+-- that holds that value. Once the module has started under the mesh on it, the value has done its
+-- work, and the mesh puts one of its own in its place — made, sealed and sent as any value it makes,
+-- so nothing a person ever handled is left in force.
+--
+-- When the value was given, for a given value awaiting that replacement; null for every other row —
+-- a value the mesh made, one an outside party issued, one a module applies to a backend, and every
+-- value given before this column existed, which `secret rotate` replaces when a person asks. The
+-- replacement waits for a clean report of a declaration sent after this moment, so it is the start
+-- on the given value that is waited for, not an older one; and it clears the column, so it happens
+-- once.
+alter table module_secret add column replace_after_start timestamptz;

internal/inventory/secrets.go

@@ -373,7 +373,8 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
 		   origin = excluded.origin, made_at = now(),
-		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
+		   replace_after_start = null`,
 		record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil {
 		return "", err
 	}
@@ -390,49 +391,61 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
 // Sealed on the way in and the plaintext discarded, exactly as a generated one is — so the only
 // difference between the two is where the value came from.
 func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, name, value string) error {
+	_, err := i.acceptOwn(ctx, node, module, name, value, false)
+	return err
+}
+
+// acceptOwn is AcceptSecretForModule, and — with untilStart, for a value a person gave to a secret
+// the mesh may make — marks it to be replaced after the module's first good start (novox/hq ADR
+// 0228). It answers whether it was so marked.
+func (i *Inventory) acceptOwn(ctx context.Context, node, module, name, value string, untilStart bool) (bool, error) {
 	// Refused for a name the module does not declare. A value stored under a name nothing reads
 	// is a delivery that changed nothing and reported success — the shape of failure the mesh
 	// is built to refuse (novox/hq 04-ISSUES/078).
 	m, err := i.declared(ctx, module)
 	if err != nil {
-		return err
+		return false, err
 	}
-	if _, own := m.OwnSecrets[name]; !own {
-		return fmt.Errorf("%s does not declare %q as an own secret; %s — a secret it requires from a provider is accepted with `--provider <node> [--local <name>]`, the value the running service already uses (novox/hq ADR 0163)", module, name, declaresOwn(m))
+	own, declared := m.OwnSecrets[name]
+	if !declared {
+		return false, fmt.Errorf("%s does not declare %q as an own secret; %s — a secret it requires from a provider is accepted with `--provider <node> [--local <name>]`, the value the running service already uses (novox/hq ADR 0163)", module, name, declaresOwn(m))
 	}
+	untilStart = untilStart && own.MeshMayMake()
 	key, err := i.SealingKeyOf(ctx, node)
 	if err != nil {
-		return err
+		return false, err
 	}
 	if key == "" {
-		return fmt.Errorf(
+		return false, fmt.Errorf(
 			"%s has no sealing key, so nothing can be sealed to it — it joins again to get one",
 			node)
 	}
 	record, err := i.NodeByName(ctx, node)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	sealed, err := secrets.Accept(value, key, key)
 	if err != nil {
-		return err
+		return false, err
 	}
 	// And to the operator, when the mesh has one: a value a person supplied is the one a person
 	// most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended).
 	forOperator, operatorKey, err := i.operatorSeal(ctx, value)
 	if err != nil {
-		return err
+		return false, err
 	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
-		 values ($1, $2, $3, $4, $5, 'accepted', $6, $7)
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key,
+		                            replace_after_start)
+		 values ($1, $2, $3, $4, $5, 'accepted', $6, $7, case when $8 then now() end)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
 		   origin = excluded.origin, made_at = now(),
-		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
-		record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey)
-	return err
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
+		   replace_after_start = excluded.replace_after_start`,
+		record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey, untilStart)
+	return untilStart, err
 }
 
 // Holder is one end-to-end credential: who gets it and who must create it.
@@ -537,8 +550,12 @@ func (e ErrNotRotatable) Error() string { return e.Why }
 // backend that takes it once would, rotated this way, leave the backend on the old value and the
 // module reading the new one — the fault issue 179 was. That form is staged, which the mesh does
 // not build yet, and is refused by name. A secret whose manifest says neither is refused with the
-// word to write; a secret given to the mesh rather than made by it is refused as 0113 says: the
-// mesh will not replace what it cannot read.
+// word to write.
+//
+// **A value given to the mesh is replaced like one it made** (novox/hq ADR 0228, extending 0113):
+// the old value is not read, and need not be — a secret the module reads at start is held by
+// nobody but that module, so a fresh one sent to it is the whole change. Refused only for a value
+// an outside party issued (`"issued-by": "outside"`): no value the mesh makes would work there.
 func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name string) error {
 	m, err := i.declared(ctx, module)
 	if err != nil {
@@ -548,6 +565,17 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
 	if !declared {
 		return fmt.Errorf("%s does not declare %q as an own secret; %s — a secret it requires from a provider is accepted with `--provider <node> [--local <name>]`, the value the running service already uses (novox/hq ADR 0163)", module, name, declaresOwn(m))
 	}
+	record, err := i.NodeByName(ctx, node)
+	if err != nil {
+		return err
+	}
+	if own.IssuedBy == catalogue.IssuedOutside {
+		return ErrNotRotatable{Why: fmt.Sprintf(
+			"%s's %q is issued by a party outside the mesh — its definition says \"issued-by\": "+
+				"\"outside\" — so no value the mesh makes would work in its place (ADR 0228)%s. Have its "+
+				"issuer make a new one, then `secret accept %s %s %s --from <file>`",
+			module, name, i.givenAgo(ctx, record.ID, module, name), node, module, name)}
+	}
 	switch own.Taken {
 	case catalogue.TakenAtStart:
 	case catalogue.TakenApplied:
@@ -564,10 +592,6 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
 				"starts, or \"applied\" when its own code applies it",
 			module, name, name)}
 	}
-	record, err := i.NodeByName(ctx, node)
-	if err != nil {
-		return err
-	}
 	key, err := i.SealingKeyOf(ctx, node)
 	if err != nil {
 		return err
@@ -585,16 +609,16 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
 	if err != nil {
 		return err
 	}
-	if origin == OriginAccepted {
-		return ErrNotRotatable{Why: fmt.Sprintf(
-			"%s on %s holds %q as a value given to the mesh, not made by it, and the mesh will not "+
-				"replace what it cannot read (ADR 0113). Change it where it lives, then `secret accept "+
-				"%s %s %s` with the new value",
-			module, node, name, node, module, name)}
-	}
+	return i.remakeOwn(ctx, record.ID, key, m, module, name)
+}
+
+// remakeOwn puts a value the mesh makes in a module's own secret, whatever it held: sealed to the
+// machine and the operator, origin made, and no longer awaiting a replacement. A secret that is a
+// provider's one credential (ADR 0158) is remade for every holder at once.
+func (i *Inventory) remakeOwn(ctx context.Context, nodeID any, key string, m catalogue.Manifest, module, name string) error {
 	if len(m.ProvisionsSharing(name)) > 0 {
 		// Shared with every consumer of those provisions (ADR 0158): one new value, sealed to all.
-		return i.remakeShared(ctx, record.ID, key, module, name, "", nil, "", "", "")
+		return i.remakeShared(ctx, nodeID, key, module, name, "", nil, "", "", "")
 	}
 	operator, err := i.OperatorKey(ctx)
 	if err != nil {
@@ -607,9 +631,9 @@ func (i *Inventory) RotateModuleSecret(ctx context.Context, node, module, name s
 	forOperator, operatorKey := operatorColumns(operator, blob)
 	_, err = i.store.Pool().Exec(ctx,
 		`update module_secret set sealed = $4, node_key = $5, origin = 'made', made_at = now(),
-		        operator_sealed = $6, operator_key = $7
+		        operator_sealed = $6, operator_key = $7, replace_after_start = null
 		  where node = $1 and module = $2 and name = $3`,
-		record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey)
+		nodeID, module, name, made.ForConsumer, key, forOperator, operatorKey)
 	return err
 }
 
@@ -725,7 +749,7 @@ func (i *Inventory) remakeShared(ctx context.Context, providerID any, providerKe
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key, origin = 'made', made_at = now(),
 		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key,
-		   generation = excluded.generation`,
+		   generation = excluded.generation, replace_after_start = null`,
 		providerID, providerModule, own, ownSealed, providerKey, forOperator, operatorKey, generation); err != nil {
 		return err
 	}

internal/inventory/secrets_test.go

@@ -789,9 +789,9 @@ func TestADeliveredPairCredentialIsRefusedForARequirementTheModuleDoesNotHave(t
 }
 
 // A module's own secret rotates when its definition says the module reads it at start: made anew,
-// sealed to the machine and the operator, origin made. Refused with the reason when the definition
-// says nothing, says the module applies it, or when the value was given to the mesh (novox/hq
-// ADR 0114, issue 180).
+// sealed to the machine and the operator, origin made — whether the mesh made the value or a person
+// gave it (novox/hq ADR 0228). Refused with the reason when the definition says nothing or says the
+// module applies it (novox/hq ADR 0114, issue 180).
 func TestAnOwnSecretRotatesOnlyWhenTheModuleReadsItAtStart(t *testing.T) {
 	inv, ctx := twoNodesWithKeys(t)
 	m := catalogue.Manifest{Module: "idp", Version: "1", OwnSecrets: catalogue.OwnSecrets{
@@ -833,12 +833,23 @@ func TestAnOwnSecretRotatesOnlyWhenTheModuleReadsItAtStart(t *testing.T) {
 		t.Fatalf("a secret that says nothing of how it is taken must be refused: %v", err)
 	}
 
+	// A value given to the mesh for a secret read at start is replaced like one it made (ADR 0228).
 	if err := inv.AcceptSecretForModule(ctx, "consumer", "idp", "session", "the-real-one"); err != nil {
 		t.Fatal(err)
 	}
-	err = inv.RotateModuleSecret(ctx, "consumer", "idp", "session")
-	if !errors.As(err, &refused) || !strings.Contains(err.Error(), "given to the mesh") {
-		t.Fatalf("an accepted value must be refused: %v", err)
+	if err := inv.RotateModuleSecret(ctx, "consumer", "idp", "session"); err != nil {
+		t.Fatalf("a given value read at start must rotate: %v", err)
+	}
+	if origin, _, err := inv.OwnSecretOrigin(ctx, "consumer", "idp", "session"); err != nil || origin != OriginMade {
+		t.Fatalf("a rotated given value is the mesh's own from then on: %q %v", origin, err)
+	}
+	// A given value the module applies stays refused, with the reason.
+	if err := inv.AcceptSecretForModule(ctx, "consumer", "idp", "admin", "the-real-one"); err != nil {
+		t.Fatal(err)
+	}
+	err = inv.RotateModuleSecret(ctx, "consumer", "idp", "admin")
+	if !errors.As(err, &refused) || !strings.Contains(err.Error(), "staged") {
+		t.Fatalf("a given value the module applies must be refused: %v", err)
 	}
 	if err := inv.RotateModuleSecret(ctx, "consumer", "idp", "nothing"); err == nil || !strings.Contains(err.Error(), "does not declare") {
 		t.Fatalf("an undeclared secret: %v", err)

internal/link/events.go

@@ -63,6 +63,9 @@ const (
 	// KeyBuiltBefore: a build the mesh already held, for a catalogue that asked what it missed. Not
 	// `built` — that is the build machine's, said as it happens, and a replay is neither.
 	KeyBuiltBefore = "built-before"
+	// KeySecretReplaced: a value given to the mesh by hand was replaced with one it made, after its
+	// module's first good start (novox/hq ADR 0228). Never the value.
+	KeySecretReplaced = "secret-replaced"
 )
 
 // Applied is what a machine now runs, as the mesh states it.