mesh-knowledge: refresh repairs records the knowledge base drafted, keeping their clauses while their source is unchanged; ADR 0114 and 0148 as fixtures (review of #213)

modules/mesh-knowledge/README.md

@@ -115,12 +115,16 @@ no counter moves. A second run changes nothing and says so. `dry-run` writes not
 `mesh-knowledge dry-run <hq checkout>` gives the same dry run over a checkout on disk, against an empty knowledge
 base.
 
-**`refresh`** (the import's third argument) repairs what an earlier import lost or misread (hq issue 487). For a record held whose every version
-was written by the import or a refresh (no one has changed it since, and nothing waits over it), it reads the
-record again at the commit and, when what it holds differs, writes it as a new version: "refreshed from hq at
-<commit>: <path>", naming each field that differs. A record anyone changed after its import (drafted, accepted
-over, corrected) is never touched, and its row says so. A record read again and found the same is left as it is,
-so a second refresh changes nothing. With `dry-run` it says, record by record, what it would change.
+**`refresh`** (the import's third argument) repairs what an earlier import lost or misread (hq issue 487). It
+takes a record held whose every version was written by the import, a refresh, or the knowledge base's own
+drafting and confirming of its clauses (nothing waits over it), reads it again at the commit and, when what it
+holds differs, writes it as a new version: "refreshed from hq at <commit>: <path>", naming each field that
+differs. It keeps what the knowledge base made of the record: written clauses with their statuses when hq writes
+the same ones, drafted or confirmed clauses (and their drafting) while the text they were drafted from is what hq
+says (else they are drafted again), and the clause a change of hq's names once judged. A record anyone changed
+after its import (returned, corrected, accepted over) is never touched, and its row says so. A record read again
+and found the same is left as it is, so a second refresh changes nothing. With `dry-run` it says, record by
+record, what it would change.
 
 A decision whose Decision section's own top-level list writes numbered statements with a bold lead gets them as
 `written` clauses. A numbered list inside a `###` subsection of the Decision section is that subsection's prose,

modules/mesh-knowledge/cmd/mesh-knowledge/fields_test.go

@@ -24,7 +24,7 @@ func imported487(t *testing.T, s Store) *world {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if out.Refused != 0 || out.Created != 13 {
+	if out.Refused != 0 || out.Created != 15 {
 		t.Fatalf("%s %+v", out.Said, out.Rows)
 	}
 	return w
@@ -107,7 +107,7 @@ func TestTheImportHoldsEveryFieldOfARecord(t *testing.T) {
 func TestClausesComeOnlyFromTheDecisionSectionsOwnList(t *testing.T) {
 	eachStore(t, func(t *testing.T, s Store) {
 		imported487(t, s)
-		for _, n := range []int{259, 272, 273, 278, 288, 40} {
+		for _, n := range []int{259, 272, 273, 278, 288, 40, 114, 148} {
 			r := mustGet(t, s, Key{KindDecision, n}).Current
 			if len(r.Clauses) != 0 || r.Drafting == nil || r.Drafting.State != DraftWaiting {
 				t.Errorf("ADR %04d: %d clause(s) %v, drafting %+v", n, len(r.Clauses), r.Clauses, r.Drafting)
@@ -254,12 +254,11 @@ func TestARefreshRepairsOnlyRecordsNobodyChangedSinceTheImportAndChangesNothingT
 				t.Fatal(err)
 			}
 		}
-		// ADR 0085 was drafted since its import: someone changed it, and the refresh never touches it.
+		// ADR 0085 was corrected by someone since its import, and the refresh never touches it.
 		e85 := mustGet(t, s, Key{KindDecision, 85})
 		d := e85.Current.Clone()
-		d.Clauses = []Clause{{N: 1, Text: "**Secret-holding is a module.** It is.", Status: ClauseInForce, Source: SourceDrafted}}
-		d.Drafting = &Drafting{State: DraftDrafted}
-		if err := s.Save(ctx, Version{By: "the agent on test (sonnet)", At: at, Why: "1 clauses drafted", Record: d}, true, false, e85.Revision); err != nil {
+		d.Title = "A secret is a provision, corrected"
+		if err := s.Save(ctx, Version{By: "someone@g14", At: at, Why: "a correction by hand", Record: d}, true, false, e85.Revision); err != nil {
 			t.Fatal(err)
 		}
 		before := countVersions(t, s)
@@ -269,7 +268,7 @@ func TestARefreshRepairsOnlyRecordsNobodyChangedSinceTheImportAndChangesNothingT
 		if err != nil {
 			t.Fatal(err)
 		}
-		if countVersions(t, s) != before || dry.Refreshed != 12 || dry.ChangedSince != 1 || dry.Created != 0 {
+		if countVersions(t, s) != before || dry.Refreshed != 14 || dry.ChangedSince != 1 || dry.Created != 0 {
 			t.Fatalf("dry run: %s", dry.Said)
 		}
 		says := map[string]ImportRow{}
@@ -290,7 +289,7 @@ func TestARefreshRepairsOnlyRecordsNobodyChangedSinceTheImportAndChangesNothingT
 		if err != nil {
 			t.Fatal(err)
 		}
-		if out.Refreshed != 12 || out.ChangedSince != 1 || countVersions(t, s) != before+12 {
+		if out.Refreshed != 14 || out.ChangedSince != 1 || countVersions(t, s) != before+14 {
 			t.Fatalf("refresh: %s", out.Said)
 		}
 		e := mustGet(t, s, Key{KindDecision, 259})
@@ -308,7 +307,7 @@ func TestARefreshRepairsOnlyRecordsNobodyChangedSinceTheImportAndChangesNothingT
 		if r := mustGet(t, s, Key{KindResearch, 21}).Current; len(r.Touches) != 4 {
 			t.Fatalf("research 021 not repaired: %v", r.Touches)
 		}
-		if e := mustGet(t, s, Key{KindDecision, 85}); e.Versions != 2 || len(e.Current.Clauses) != 1 {
+		if e := mustGet(t, s, Key{KindDecision, 85}); e.Versions != 2 || e.Current.Title != "A secret is a provision, corrected" {
 			t.Fatalf("0085, changed since its import, was touched: %d versions", e.Versions)
 		}
 		// Run again: nothing changes, and it says so.
@@ -317,22 +316,81 @@ func TestARefreshRepairsOnlyRecordsNobodyChangedSinceTheImportAndChangesNothingT
 		if err != nil {
 			t.Fatal(err)
 		}
-		if again.Refreshed != 0 || again.Unchanged != 12 || countVersions(t, s) != after || !strings.Contains(again.Said, "Nothing changed") {
+		if again.Refreshed != 0 || again.Unchanged != 14 || countVersions(t, s) != after || !strings.Contains(again.Said, "Nothing changed") {
 			t.Fatalf("a second refresh: %s", again.Said)
 		}
-		// Once drafted, a repaired record is someone's: a later refresh leaves it alone.
-		draftUntil(t, w, Key{KindDecision, 259}, DraftDone)
+		// Drafted and confirmed since, by the knowledge base itself: still the import's side. A later refresh keeps
+		// the drafted clauses, since the Decision section they come from is unchanged, and changes nothing.
+		r259 := draftUntil(t, w, Key{KindDecision, 259}, DraftDone)
 		third, _ := w.k.Import("import@test", ImportAsk{Refresh: true}, at487())
-		changed := false
 		for _, row := range third.Rows {
-			changed = changed || (row.Record == "ADR 0259" && row.Action == ImportChangedSince)
+			if row.Record == "ADR 0259" {
+				t.Fatalf("0259 in a third refresh: %+v", row)
+			}
 		}
-		if third.Refreshed != 0 || !changed || mustGet(t, s, Key{KindDecision, 259}).Current.Drafting.State != DraftDone {
+		if got := mustGet(t, s, Key{KindDecision, 259}).Current; third.Refreshed != 0 || renderJSON(got.Clauses) != renderJSON(r259.Clauses) {
 			t.Fatalf("a third refresh: %s", third.Said)
 		}
 	})
 }
 
+// Live, ADR 0002 has versions 2 and 3 drafted and confirmed by the knowledge base after its import. Those are the
+// import's side: a refresh repairs it, keeping its clauses while the Decision section they were drafted from is
+// what hq says, and queuing it to be drafted again when it is not.
+func TestARefreshRepairsARecordTheKnowledgeBaseDraftedAndKeepsItsClauses(t *testing.T) {
+	eachStore(t, func(t *testing.T, s Store) {
+		ctx := context.Background()
+		w := newWorld(t, s, at487())
+		commit, _ := at487().Standing()
+		p2 := DecisionsFolder + "/0002-nodes-communicate-over-a-broker.md"
+		p4 := DecisionsFolder + "/0004-a-node-and-how-it-joins.md"
+		r2 := legacy(t, p2, commit)
+		// 0004 has no Decision section; the import before #487 queued nothing, so a hand-made drafting stands in for
+		// one made from a text that is no longer what hq says.
+		r4 := legacy(t, p4, commit)
+		r4.Drafting = &Drafting{State: DraftWaiting}
+		r4.Sections = append(r4.Sections, Section{Name: "Decision", Text: "Nodes join by a token. That is decided here."})
+		for _, c := range []struct {
+			path string
+			r    *Record
+		}{{p2, r2}, {p4, r4}} {
+			if err := s.Create(ctx, Version{By: Unproven, Why: "imported from hq at " + short(commit) + ": " + c.path, Record: c.r}); err != nil {
+				t.Fatal(err)
+			}
+		}
+		drafted2 := draftUntil(t, w, Key{KindDecision, 2}, DraftDone)
+		draftUntil(t, w, Key{KindDecision, 4}, DraftDone)
+		if mustGet(t, s, Key{KindDecision, 2}).Versions != 3 {
+			t.Fatal("0002 was not drafted and confirmed as live")
+		}
+		out, err := w.k.Import("import@test", ImportAsk{Refresh: true}, at487())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if out.ChangedSince != 0 {
+			t.Fatalf("drafting counted as a change: %s %+v", out.Said, out.Rows)
+		}
+		e2 := mustGet(t, s, Key{KindDecision, 2})
+		if e2.Versions != 4 || e2.Current.Reconstructed == nil || !*e2.Current.Reconstructed ||
+			renderJSON(e2.Current.Clauses) != renderJSON(drafted2.Clauses) || e2.Current.Drafting == nil || e2.Current.Drafting.State != DraftDone {
+			t.Fatalf("0002 after the refresh: %d versions, %+v, drafting %+v", e2.Versions, e2.Current.Clauses, e2.Current.Drafting)
+		}
+		// 0004's clauses were drafted from a text hq does not hold: taken back, and drafted again from its sections.
+		e4 := mustGet(t, s, Key{KindDecision, 4}).Current
+		if len(e4.Clauses) != 0 || e4.Drafting == nil || e4.Drafting.State != DraftWaiting || !strings.Contains(e4.Drafting.Reason, "no Decision section") {
+			t.Fatalf("0004 after the refresh: %+v %+v", e4.Clauses, e4.Drafting)
+		}
+		// A record someone returned is someone's change: left alone.
+		if _, err := w.k.Return("operator@shanks", Key{KindDecision, 2}, "clause 2 says more than the record"); err != nil {
+			t.Fatal(err)
+		}
+		again, _ := w.k.Import("import@test", ImportAsk{Refresh: true}, at487())
+		if again.ChangedSince != 1 {
+			t.Fatalf("a returned record refreshed: %s", again.Said)
+		}
+	})
+}
+
 // A refresh without a commit's records refuses, and an import without refresh never rewrites what it holds.
 func TestAnImportWithoutRefreshLeavesAHeldRecordAsItIs(t *testing.T) {
 	w := newWorld(t, newMemStore(), at487())

modules/mesh-knowledge/cmd/mesh-knowledge/import.go

@@ -314,21 +314,79 @@ func (k *KB) Import(caller string, a ImportAsk, records Records) (*Imported, err
 	return out, nil
 }
 
-// importedOnly says whether every version of a record was written by the import or a refresh of it: nobody has
-// changed it since it was read from hq.
+// draftedHere is the why of a version the knowledge base wrote itself while drafting and confirming clauses, or
+// judging which clause a change of hq's names (draft.go): the import's side of a record, not anyone's change.
+var draftedHere = regexp.MustCompile(`^(?:\d+ clauses drafted by |\d+ of \d+ drafted clauses confirmed by |drafting failed: |` +
+	`confirming failed: |no Decision section|which clause of .+ it changes drafted: |that it changes .+ (?:was not confirmed: |confirmed by )|` +
+	`§\d+ changed by .+ \(as hq said, its clause judged and confirmed\)$)`)
+
+// importedOnly says whether every version of a record was written by the import, a refresh of it, or the
+// knowledge base's own drafting and confirming of its clauses: nobody has changed it since it was read from hq. A
+// return, a correction, an acceptance over it or an insight is someone's change.
 func importedOnly(vs []Version) bool {
 	if len(vs) == 0 {
 		return false
 	}
 	for _, v := range vs {
-		if v.Insight || v.Record == nil || !strings.HasPrefix(v.Record.Source, "hq@") ||
-			!(strings.HasPrefix(v.Why, "imported from hq at ") || strings.HasPrefix(v.Why, "refreshed from hq at ")) {
+		if v.Insight || v.Record == nil || !strings.HasPrefix(v.Record.Source, "hq@") {
+			return false
+		}
+		imported := strings.HasPrefix(v.Why, "imported from hq at ") || strings.HasPrefix(v.Why, "refreshed from hq at ")
+		drafted := draftedHere.MatchString(v.Why) && (v.By == "the knowledge base" || strings.HasPrefix(v.By, "the agent on "))
+		if !imported && !drafted {
 			return false
 		}
 	}
 	return true
 }
 
+// carry keeps in a record read again (now) what the knowledge base made of the one held: its clauses with their
+// statuses when hq writes the same ones; its drafted or confirmed clauses and their drafting while the text they
+// were drafted from is what hq says (else they are drafted again); and the clause a change of hq's names, once
+// judged.
+func carry(held, now *Record) {
+	switch {
+	case len(now.Clauses) > 0:
+		if len(held.Clauses) == len(now.Clauses) {
+			same := true
+			for i := range now.Clauses {
+				same = same && held.Clauses[i].N == now.Clauses[i].N && held.Clauses[i].Text == now.Clauses[i].Text
+			}
+			if same {
+				now.Clauses = held.Clauses
+			}
+		}
+	case now.Drafting != nil && held.Drafting != nil && (len(held.Clauses) > 0 || held.Drafting.State != DraftWaiting):
+		a, _ := draftSource(held)
+		b, _ := draftSource(now)
+		if a == b {
+			now.Clauses, now.Drafting = held.Clauses, held.Drafting
+		}
+	}
+	judged := map[Ref][]Link{}
+	for _, l := range held.Links {
+		if l.Kind == LinkChanges && l.Source != LinkPending {
+			if to, ok := l.To.Record(); ok {
+				judged[Ref{Kind: to.Kind, Number: to.Number}] = append(judged[Ref{Kind: to.Kind, Number: to.Number}], l)
+			}
+		}
+	}
+	var links []Link
+	for _, l := range now.Links {
+		target := Ref{Kind: l.To.Kind, Number: l.To.Number}
+		if l.Kind == LinkChanges && l.Source == LinkPending && len(judged[target]) > 0 {
+			for _, j := range judged[target] {
+				j.Unresolved = l.Unresolved
+				links = append(links, j)
+			}
+			delete(judged, target)
+			continue
+		}
+		links = append(links, l)
+	}
+	now.Links = links
+}
+
 // refresh reads a held record again (r, as hq has it at the commit) and, when only the import ever wrote it and
 // what it holds differs, writes r as its new version naming what changed and why. A record changed here since
 // its import is never touched; a second refresh finds the same and changes nothing.
@@ -349,6 +407,7 @@ func (k *KB) refresh(ctx context.Context, a ImportAsk, e *Entry, r *Record, path
 		out.ChangedSince++
 		return
 	}
+	carry(e.Current, r)
 	diffs := differences(e.Current, r)
 	if len(diffs) == 0 {
 		row.Action = ImportHeld

modules/mesh-knowledge/cmd/mesh-knowledge/testdata/hq-f0892019/02-DECISIONS/0114-a-shared-credential-rotates-over-two-credentials.md

@@ -0,0 +1,270 @@
+---
+topic: what runs on it
+status: accepted
+date: 2026-09-26
+deciders: jochen
+reconstructed: false
+extends: 02-DECISIONS/0113-the-vault-makes-every-secret.md
+---
+
+# 114. A credential two parties hold rotates over two credentials; one a single party holds rotates in place, staged; and retiring a credential never removes what it reached
+
+## Context
+
+[ADR 0113](0113-the-vault-makes-every-secret.md) decides who asks for a rotation (an operator, or the
+vault's policy) and who makes the new value (the vault). It leaves open how old and new change over.
+[Research 016](../01-RESEARCH/016-how-a-credential-can-be-rotated/00-overview.md) read every
+credential provider in the catalogue against its code. There are nine:
+
+- **all nine re-apply a password in place**, on the same login, every time they run. The controller's
+  `rotate` command relies on that, and states the window it leaves: between the provider applying the
+  new value and the consumer restarting with it, the consumer cannot authenticate;
+- **eight of nine name the consumer's resource after its login**: a database, a bucket, a virtual host,
+  a key prefix, a topic prefix, a mailbox. Only the forge's npm registry keeps them apart, because an
+  organisation owns the packages;
+- **five of nine destroy the consumer's data when they remove its login**: postgres, mssql and mongodb
+  drop the database, lavinmq drops the virtual host with its queued messages, and mailu deletes the
+  mailbox with its mail. In those adapters, *retire a login* and *delete the consumer's data* are one
+  call. minio drops a bucket only if it is empty. The provisioner harness makes it worse: a consumer
+  whose derived login changed is removed under the old login and created under the new one, in one pass;
+- **one backend holds two passwords on one login** (redis), and two hold several tokens beside one
+  password (the forge and mailu);
+- **eight of nine can give two logins the same rights over one resource**. mailu cannot, because a mail
+  user *is* its mailbox. It can give one user several tokens. In postgres, a second login is not enough
+  on its own: objects belong to whichever login created them, so the resource must be owned by a role of
+  its own;
+- **an administrative credential has one party and a fixed name.** The provider module both applies it
+  and reads it. Five backends take it only at first initialisation: postgres, mssql, mongodb, mosquitto
+  and lavinmq. Their credential file is mounted directly into both the server and the provisioner, so
+  replacing the file recreates the provisioner holding only the new value, which the backend does not
+  know yet. The provisioner is then locked out;
+- **no module watches a secret.** Every reader reads at start, and the host recreates a container when
+  a file it read at creation changes
+  ([issue 103](../04-ISSUES/103-a-container-is-not-recreated-when-a-file-it-reads-changes/00-report.md)).
+
+A first draft of 0113 chose to overlap old and new "through the adapter's existing create and
+remove". In five providers, that remove deletes the consumer's data. The mechanism has to be chosen on
+what the providers do, and the danger has to be closed whichever mechanism is chosen.
+
+## Considered Options
+
+**1. In place for everything, as today.** Works with every provider unchanged. Rejected for credentials
+two parties hold. The window cannot be closed, only shortened, and the two ends are on different
+machines with nothing ordering them. For an administrative credential, it locks the provisioner out.
+
+**2. Two secrets on one login.** Rejected as the mechanism. It works for three providers out of nine,
+and using it there and something else elsewhere would put the difference in the mesh instead of in the
+adapter.
+
+**3. Two logins over one resource.** Rejected as the mechanism. It works for eight of nine, and not for
+mailu.
+
+**4. Two credentials over one resource, with the adapter choosing what a credential is.** A credential
+is what a consumer presents, a login and a secret. The mesh alternates between two of them. Each adapter
+makes the second one the way its backend can: a second login for eight providers, a second token on the
+same login for mailu. A credential a single party holds is staged in place instead, and retiring a
+credential is separated from removing a consumer before either is used. Chosen.
+
+## Decision
+
+### Retiring a credential never removes what it reached
+
+**A provider's adapter keeps two things apart that today are one:** the consumer's *resource* (its
+database, bucket, virtual host, key or topic prefix, mailbox) and a *credential* that reaches it.
+They get separate operations:
+
+- **ensure the resource**, named after the consumer;
+- **ensure a credential** with a value, holding the consumer's rights over its resource;
+- **retire a credential**. Anything it owns moves first to the resource's owner, and any session it has
+  open is ended. Then the credential is removed, and nothing else;
+- **remove the consumer**, which is what removes the resource, and retires every credential it has.
+
+**Remove the consumer runs only when the consumer no longer requires the provision from this provider.**
+That happens when its assignment goes, when its definition drops the requirement, or when re-resolution
+sends it to another provider. It never runs because a login or a value changed. The harness keys what it
+applied by the consumer, not by the login, so a changed login is a credential change and never a removal.
+What removing a resource does with the data in it stays
+[ADR 0030](0030-data-outlives-the-mesh-that-declared-it.md)'s, and re-resolving to another provider
+moves no data.
+
+**The resource is named after the consumer, and owned by the resource, not by a login.** A consumer's
+identity is derived from its assignment ([ADR 0049](0049-a-consumers-identity-fits-the-tightest-backend.md)),
+and today its login is that same string, so **no existing resource is renamed**. Where a backend makes
+whatever a login creates the login's own, as postgres does, the resource is owned by a role that cannot
+log in, and each credential works as that role. Ownership of an existing resource moves to it once. A
+credential is retired by handing what it owns to that role, never by dropping what it owns.
+
+### A credential two parties hold rotates over two credentials
+
+**Two parties** means an applier and a reader that are different modules, or a module and a node's
+host. The vault's custody copy does not count, because the vault holds every secret. So this covers a
+credential between a consumer and a provider, and every bus account: a module's or a host's, applied by
+the broker's provisioner and read by its owner. **Each consumer has two credentials, one in use at a
+time**, both holding the same rights over the one resource. For eight providers the second is a second
+login, derived by the mesh as the consumer's identity with a short fixed suffix. For mailu it is a
+second token on the same login.
+
+**The vault drives each rotation and records every step durably.** A provisioner learns which
+credentials to hold from what it receives: both of them, for as long as a rotation is under way. It
+never learns them from its own memory, so a provisioner restarted mid-rotation resumes from the step the
+vault has recorded.
+
+1. **The vault makes the new value.**
+2. **Each applier ensures the unused credential with it**, with the consumer's rights, and leaves the
+   one in use untouched. It verifies that the new credential authenticates and the old one still does,
+   and confirms. It repeats the confirmation on every reconcile pass until the vault acknowledges it, so
+   a lost message costs one pass.
+3. **Only then does the vault release the new credential to the readers.** The mesh delivers it and the
+   value together, and the host recreates each reader, because a file it read at creation changed. A
+   node's host is its own reader: it reconnects to the bus with the new login, and confirms over it.
+4. **Each reader confirms by authenticating with the new credential.** It shows this through its
+   health check, where its definition declares one, or the applier sees the new credential in use,
+   where its backend reports that. A reader for which neither is possible is confirmed by an operator.
+   It is never assumed from the reader having restarted.
+5. **Only when every reader has confirmed is the old credential retired**, as above, and verified to no
+   longer authenticate.
+
+**A reader that goes away leaves the rotation.** A reader unassigned, or re-resolved to another
+provider, is no longer waited for. A consumer removed mid-rotation has both of its credentials retired
+with it.
+
+**A rotation can be abandoned until the old credential is retired.** An operator abandons it. Readers
+that moved are given the old credential back, and recreated. The new credential is retired. Nothing is
+lost, because the old one was never removed.
+
+`status` shows a rotation as waiting on whichever applier or reader has not moved, and it is not done
+until the old credential is gone. A reader that cannot be reached keeps working on the old credential
+until it can, and the rotation waits for it. That wait is shown, never hidden.
+
+**Queues and permissions belong to the consumer, not to a login.** A module's queue on the bus is named
+for the module on its node, and both of its logins get the same permissions over it
+([ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md)). An MQTT client
+identifier is chosen by the consumer and is independent of its login. A reader recreated with a new
+login keeps it, and the broker hands the session over.
+
+### A credential a single party holds rotates in place, staged
+
+This covers a provider's administrative credential and a module's own secret, which only that module
+reads. The vault makes the new value, and the one party takes it:
+
+- **applied**: the vault delivers the new value **staged, beside the current one**, and the current file
+  is left as it is. The party's provisioner changes the backend using the current value, verifies the
+  new one, and confirms. Only then does the vault make the new value current. This is the only form for
+  a backend that takes its administrative credential only at first initialisation. Replacing the file
+  first would lock the provisioner out;
+- **read at start**: the vault delivers the new value as current, and the host recreates the party.
+
+There is no window between two parties, because there is only one. Where neither form can change the
+value, the requirement is marked not rotatable by the mesh, and a rotation is refused, saying why
+([ADR 0113](0113-the-vault-makes-every-secret.md)).
+
+### One rule decides which
+
+**The number of parties decides, never the provider.** The resolver knows it from the requirement's
+recipients, leaving out the vault's custody copy, so no definition declares it.
+
+> **Progressive insight — 2026-10-01.** The number of parties is the resolver's to know; *which form*
+> a single party's credential takes is not, and cannot be: whether a module reads its secret when it
+> starts or applies it once to a backend is a fact about the software, visible nowhere in the graph.
+> So the definition declares that half — `taken: at-start` or `taken: applied` on an own secret — and
+> a secret that declares neither is not rotated, refused with the word to write (issue 180). The
+> read-at-start form is built; the staged form for an applied credential is not. The decision stands;
+> the sentence above was one fact short.
+
+### Until an adapter can
+
+**An adapter that cannot yet ensure a second credential says so.** The two-party credentials it applies
+rotate in place, as today, and the window is stated when the rotation is asked for. So does a
+two-party credential whose backend has one fixed name and no second credential for it. These are listed
+by a check, and the list is meant to shrink. Separating *retire a credential* from *remove the
+consumer*, and keying the harness by consumer, come first. They close a data-loss path that exists
+today, whatever rotation does.
+
+## What this changes in earlier records
+
+On acceptance, each of these is amended by this record, not edited:
+
+- [ADR 0113](0113-the-vault-makes-every-secret.md): the changeover it left open is decided here.
+- [ADR 0049](0049-a-consumers-identity-fits-the-tightest-backend.md): a consumer's identity leaves room
+  for the second login's suffix within the tightest backend it reaches, and both logins are checked
+  against it.
+- [ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md): a module's broker
+  account is two logins with the same permissions over the same queue, one in use at a time. Its scoping
+  is unchanged.
+- [ADR 0048](0048-a-provider-creates-the-credential-the-mesh-minted.md), already superseded by 0113:
+  a provider now ensures and retires credentials over a resource it owns separately.
+- [To-be 13](../03-DESIGN/01-to-be/13-credentials-and-their-rotation.md): rotation of a two-party
+  credential is no longer all-or-nothing with a window. It overlaps, with each step confirmed.
+  A single-party credential is staged, not replaced.
+
+## Accepted, 2026-09-30, and not scheduled
+
+Accepted as written. The separation it draws — a consumer's *resource* and a *credential that reaches
+it* are different things with different lifecycles — is the part that had to be settled, because the
+alternative is what the record was written against: retiring a credential taking the data it reached
+with it. That is a data-loss shape, and a record that names it should not sit unresolved while the
+code that could hit it is being written.
+
+**It is not built, and accepting it does not schedule it.** The SDK's provisioner adapter is still
+`create` / `remove` / `holds` rather than the four operations above, and no provider implements the
+two-credential rotation. Accepted-and-not-built is an ordinary state here — 0141 and 0142 are both in
+it — and it is the honest one: leaving this `proposed` made it invisible to anyone reading what the
+mesh has decided, while changing nothing about what runs.
+
+The work it implies belongs with the provisioner contract, beside
+[issue 124](../04-ISSUES/124-a-consumer-cannot-be-told-what-its-provider-derived/00-report.md).
+
+## Consequences
+
+- **Every credential provider's adapter changes**, in two steps. The first separates *retire a
+  credential* from *remove the consumer*. It names and owns the resource after the consumer, which
+  keeps the name it has but moves ownership once in postgres and mssql, and the harness is keyed by
+  consumer. The second ensures a second credential with the same rights.
+- **The vault gains rotation state**: each rotation's step, per applier and reader, recorded durably.
+  Staged delivery is added for single-party secrets. The SDK harness carries the alternation and the
+  repeated confirmation, so no adapter implements them.
+- **No consumer module changes.** It reads one credential at start, as today, and is recreated by the
+  host when it changes. The exception is a reader that has neither a health check nor a backend that
+  reports use: its rotations wait for an operator until it declares one.
+- **The derived identity is two characters tighter** in the tightest backend, a minio access key of 20
+  characters.
+- **What got harder:**
+  - a provider briefly holds two credentials per consumer;
+  - a rotation lasts until its slowest reader moves, so an unreachable reader keeps the old credential
+    valid until it is reached;
+  - an adapter has four operations where it had two;
+  - retiring a login in mssql has to end its sessions first.
+
+## How it is checked
+
+| Rule | Checked by |
+|---|---|
+| Retiring a credential never removes a resource | A provider test per credential provider: retiring one of a consumer's credentials leaves its resource and data intact, reachable through the other. |
+| What a retired login owned survives it | A postgres and an mssql test: objects created under login A, tables included, are still there and alterable under login B after A is retired. |
+| A changed login is not a removal | A harness test: changing a consumer's derived login ensures a credential and never calls remove. |
+| Remove runs only when the requirement goes | Harness tests: unassigning, dropping the requirement and re-resolving each remove the consumer once; a rotation and a login change never do. |
+| No existing resource is renamed | A provider test: a consumer created before the change keeps its resource, with ownership moved to the resource's own role where the backend needs one. |
+| Both credentials hold the same rights | A provider test per credential provider: data and structure created under one credential are read, changed and altered under the other. |
+| Readers move only after the applier confirms | A rotation test: readers receive nothing until both credentials authenticate at every applier. |
+| A reader confirms by authenticating | A rotation test: a reader recreated but failing to authenticate with the new credential does not confirm, and the old credential is not retired. |
+| The old credential is retired only after every reader confirms | A rotation test with one reader's node unreachable: it keeps authenticating with the old credential, the rotation shows waiting on it, and it completes when the reader returns and confirms. |
+| A reader that goes away leaves the rotation | A rotation test: unassigning a waiting reader lets the rotation complete; removing the consumer mid-rotation retires both credentials. |
+| A rotation can be abandoned | A rotation test: abandoning after readers moved gives them the old credential back and retires the new one. |
+| Rotation state survives a restart | A test restarting the applier's provisioner, and then the vault, between steps: the rotation resumes from the recorded step. |
+| A single-party applied secret is staged | A rotation test on a first-initialisation administrative credential: the provisioner receives the new value beside the current one, applies it, and only then does the new value become current. At no point does it lose its connection. |
+| The number of parties decides | A resolution test: a secret with an applier and a reader in different parties is marked for two credentials, and one held by one module for in place. The vault's copy is not counted. |
+| Both logins fit the tightest backend | A controller test: both derived logins for the longest node and module names fit the limit ADR 0049 sets. |
+| A host rotates its bus login | A rotation test on a node's bus account: the host reconnects with the new login and confirms over the bus before the old one is retired. |
+| What still rotates in place is listed | A catalogue test lists every adapter that cannot yet ensure a second credential, and every two-party credential with one fixed name. A rotation of these states its window. |
+
+## References
+
+- [Research 016](../01-RESEARCH/016-how-a-credential-can-be-rotated/00-overview.md): the survey this
+  rests on, provider by provider
+- [ADR 0113](0113-the-vault-makes-every-secret.md): who asks and who makes
+- [ADR 0049](0049-a-consumers-identity-fits-the-tightest-backend.md), [ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md),
+  [ADR 0030](0030-data-outlives-the-mesh-that-declared-it.md): identity, bus accounts, and data outliving
+  its declaration
+- [To-be 13](../03-DESIGN/01-to-be/13-credentials-and-their-rotation.md): rotation as implemented
+- [Issue 103](../04-ISSUES/103-a-container-is-not-recreated-when-a-file-it-reads-changes/00-report.md):
+  why a reader's restart can be derived

modules/mesh-knowledge/cmd/mesh-knowledge/testdata/hq-f0892019/02-DECISIONS/0148-the-meshs-names-are-resolved-not-copied-into-containers.md

@@ -0,0 +1,176 @@
+---
+topic: the tiers
+status: accepted
+date: 2026-09-30
+deciders: jochen
+reconstructed: false
+extends: 02-DECISIONS/0066-public-routing-is-name-agnostic.md
+---
+
+# 148. The mesh's names are resolved, not copied into every container
+
+## Context
+
+The mesh gives every container it declares the whole roster of mesh names as entries written into
+the container's own hosts file at creation
+([design 08 §2](../03-DESIGN/01-to-be/08-connectivity.md),
+[ADR 0066](0066-public-routing-is-name-agnostic.md)). A container takes those entries once and never
+looks again.
+
+Three issues are the same fact arriving three times.
+
+**A container keeps the address it was made with.** Adopting the predecessor's tunnel moved the hub's
+private address; the declaration followed it within one push and nothing on the machine did. The
+forge's container held the old address, lost its database, reported healthy while its existing
+connections lasted, and then the public name went down
+([issue 109](../04-ISSUES/109-a-container-keeps-the-address-it-was-made-with/00-report.md)).
+
+**The same fault, four days later, undetected for five days.** One container had restarted 2286 times
+against a database it could no longer find, while the mesh reported the machine as doing what it was
+told. Inside it, `novox.internal` was an address that had not existed for five days
+([issue 135](../04-ISSUES/135-a-containers-mesh-names-are-not-compared/00-report.md)). Forty-eight
+other containers were current, none of them corrected — each had been recreated for some other
+reason and picked up the roster on the way.
+
+135 was fixed by putting the roster into the digest the host compares a container against, so a
+container whose names moved is recreated like one whose image moved. **That made the roster part of
+every container's identity**, which is the third arrival:
+
+**One name moving replaces every container in the mesh.** Migrating one small module on one machine
+took four routine actions; each changed the roster, and each replaced every container on the control
+node — its own store, the registry, the edge proxy, the forge, the directory, mail. The control plane
+was unreachable twice while its own store came back through crash recovery
+([issue 151](../04-ISSUES/151-a-new-name-recreates-every-container-in-the-mesh/00-report.md)). None of
+the replaced containers had anything to do with the module being migrated, or with its machine.
+
+The blast radius of a name is now every container that carries the list, which is all of them. The
+node-by-node migration ahead adds names one module at a time — on one machine alone that is around
+twenty-five — and each would be a full restart of every service on the hub.
+
+## Considered Options
+
+**1. Keep the roster in every container and accept the churn.** Rejected. It is not a cost that can
+be paid down: the mesh gets more names as it grows, and every name costs a restart of everything.
+A rollback costs another.
+
+**2. Scope each container's entries to the names it actually binds.** A container is given the names
+of the things it declared a requirement on, so a name's blast radius is its consumers. Tidy, needs no
+new mechanism, and keeps 135's guarantee exactly.
+
+Rejected, and this is the close one. It contradicts the standing intent that **anything on the mesh
+can call anything on it** — three cases, same machine, the private network, the public network, and
+no fourth. Scoping resolution to declared couplings makes a name reachable only where the mesh was
+told in advance that it would be wanted, and a person debugging inside a container would find names
+missing that exist everywhere else on the machine. It also leaves the roster in the digest, so the
+churn returns the moment a widely-bound name moves — smaller, not gone.
+
+**3. Resolve at lookup time through the machine's resolver, and copy nothing.** Chosen.
+
+## Decision
+
+**A container resolves the mesh's names through its machine's resolver, at the moment it asks. No
+mesh name and no mesh address is written into a container, and none is part of a container's
+identity.**
+
+The three consequences that make this worth doing:
+
+- **Staleness stops being possible**, rather than being detected. 109 and 135 are not bugs that were
+  fixed; they are a shape that no longer exists. A name that moves is answered differently by the next
+  lookup, in every container, with nothing recreated and nothing restarted.
+- **A name's blast radius becomes nothing.** Assigning a module on one machine does not touch a
+  container on another.
+- **Anything can still call anything**, which option 2 gave up. The resolver answers every mesh name to
+  every asker on the machine, exactly as it answers the machine itself.
+
+**The resolver is a machine-level process, not a container** — one of the modules that is not a
+container at all — so a container depending on it is not the circularity it would be if the mesh's
+own store had to resolve a name through something the store's own runtime had to start first.
+
+**What a module declares for itself is untouched.** Entries a manifest asks for are the module's own,
+stay in the container, and stay in its identity: they are part of what the module *is*, they do not
+move when the mesh's roster does, and the mesh does not know what they mean.
+
+**The machine's own roster file is untouched.** It is a file, rewritten in place, read by processes and
+people; nothing restarts when it changes. It is only the *copy into each container* that this ends.
+
+### The order this lands in, which is not a preference
+
+**Nothing may stop copying names until resolution works from a container.** Removing the copy first
+reintroduces 109 and 135 — silently, and on a live mesh, which is exactly how both were found.
+
+1. **A container on any network can reach the resolver.** Today a container on the runtime's default
+   network asks from an address the converged filter drops, so it has no DNS at all
+   ([issue 110](../04-ISSUES/110-a-container-on-the-runtimes-own-network-cannot-reach-the-resolver/00-report.md));
+   and on two of four machines the resolver binds loopback only, so the runtime hands containers a
+   public resolver instead. Both are prerequisites, not related work.
+
+   > **Progressive insight — 2026-09-30, later the same day. The loopback claim was wrong.** The
+   > resolver bound the private address on all four machines; on two the runtime had never been told
+   > to use it, and on all four the resolver discarded a query that arrived on the runtime's bridge.
+   > The step stands; the facts under it were those. Both fixed the same day
+   > ([issue 110's resolution](../04-ISSUES/110-a-container-on-the-runtimes-own-network-cannot-reach-the-resolver/01-resolution.md)),
+   > and step 3 landed after them.
+2. **The runtime is told which resolver to use, per machine, as a file** — not per container as a
+   creation-time argument, or the resolver's address is back in every container's identity and the
+   problem has only got smaller.
+3. **Then, and only then, the roster leaves the declaration and the digest.**
+
+Until step 3 the mesh keeps copying, and keeps comparing. 151 stays open until step 3 lands; it is not
+closed by this record, only answered by it.
+
+## How this is checked
+
+- **A container resolves a name that moved, without being recreated.** Move a name the mesh serves;
+  from a container that was running before the move and has not been touched since, the name answers
+  with the new address. This is the one 109 and 135 would both have failed.
+- **A name's blast radius is nothing.** Add a routed name on one machine; no container on any other
+  machine is recreated. The apply report on each machine says nothing changed. This is 151.
+- **Anything calls anything.** From a container on any machine, every `<node>.internal` name and every
+  routed name the mesh serves resolves — including names the module never declared a requirement on,
+  which is the guarantee option 2 would have given up.
+- **On every network the runtime offers.** The first three hold for a container on the runtime's
+  default network as well as one on a declared network, because the default network is the case that
+  has no DNS today.
+- **No mesh name is in a container's spec.** A test asserts the digest a host computes for a container
+  does not move when the mesh's roster does, and does move when the module's own declared entries do.
+
+## Consequences
+
+- **The resolver becomes load-bearing for every container**, where before it was load-bearing for the
+  machine. This is a real cost and is accepted: a resolver that is down is a machine that cannot
+  resolve, which is already true of the machine itself, and is a smaller event than a roster change
+  destroying and recreating every container on the machine.
+- **Design 08's "a file rather than a resolver" no longer describes containers.** It was written when
+  the mesh had no resolver and it gave the right answer then. The reasoning it rested on — every Linux
+  has a hosts file, no package needed — was already overtaken by names a hosts file cannot express:
+  service names and wildcards under `<node>.internal`, which is why the resolver was built.
+- **ADR 0066's mesh-wide propagation is kept and its mechanism changes.** A routed name still reaches
+  every asker in the mesh; it reaches them through the resolver rather than by being written into each
+  container. The consequence 0066 records — that an internal issuer's challenge needs the routed name
+  resolvable inside the mesh — holds unchanged and by the same means the machine already uses.
+- **Issue 110 stops being a container-DNS inconvenience and becomes a prerequisite** for the mesh not
+  restarting itself whenever it learns a name.
+- **A container that names a resolver of its own has opted out of the machine's**, and the copy this
+  record removes was the only reason such a container could reach anything by a mesh name.
+
+  > **Progressive insight — 2026-09-30, the afternoon this landed. Found the hard way.** The mail
+  > system's admin, behind Mailu's own resolver, lost its database the moment the copy went
+  > ([issue 171](../04-ISSUES/171-a-modules-own-resolver-knows-no-mesh-name/00-report.md)). A `dns` on
+  > a container is a decision about whether mesh names exist inside it, not a preference; the module
+  > was corrected, and whether the controller should refuse the contradiction is that issue's open
+  > question.
+
+- **A container started by hand gets the mesh's names too**, where before only declared containers did.
+  Design 08 drew that boundary deliberately, on the grounds that reaching into every container is what
+  a nameserver would be for. This record accepts that consequence rather than working around it: a
+  person debugging in a hand-started container resolving the same names as everything else is the
+  behaviour worth having, and it is what "anything can call anything" means.
+
+## References
+
+- [issue 151](../04-ISSUES/151-a-new-name-recreates-every-container-in-the-mesh/00-report.md) — one name replaces every container; the question this answers
+- [issue 135](../04-ISSUES/135-a-containers-mesh-names-are-not-compared/00-report.md) — the roster put into the digest
+- [issue 109](../04-ISSUES/109-a-container-keeps-the-address-it-was-made-with/00-report.md) — the first arrival
+- [issue 110](../04-ISSUES/110-a-container-on-the-runtimes-own-network-cannot-reach-the-resolver/00-report.md) — the prerequisite
+- [ADR 0066](0066-public-routing-is-name-agnostic.md) — routed names propagate mesh-wide; extended here
+- [design 08 §2](../03-DESIGN/01-to-be/08-connectivity.md) — the file-not-resolver reasoning this narrows