Raise genesis on the bus the mesh runs on, from one bundle (hq issue 146, ADR 0331)

The installer could not raise a first node: the old bundle's control plane had no MESH_BUS_NATS and crash-looped, and the NATS bundle was refused because rootsecrets only knew the previous broker's guest:guest@ template. One bundle now, the NATS one. rootsecrets makes the controller's bus account and puts it in both placeholders (the address and the bcrypt hash in the carried user list), kept with its hash so a re-run writes the same bundle. Preflight refuses a template naming no bus connection it knows, naming the placeholder, instead of waiting out the verify step. The broker step adopts the bus as the nats module and refuses, by name, a catalogue without it or a server that is not the foundation's container.
2026-10-12 22:49:49 +02:00
parent d8e785bb4b
commit 37947ca4ee
19 changed files with 681 additions and 530 deletions
@@ -5,13 +5,24 @@ import (
 	"strings"
 	"testing"

+	"golang.org/x/crypto/bcrypt"
+
 	"github.com/novox/mesh-host/internal/declaration"
 )

-// The produced bundle carries no well-known credential: the store reads its password from the
-// file genesis made, every connection string names the made values, and the broker's default
-// administrator is changed by an action before anything dials it (novox/hq issue 071).
-func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
+// testCredentials are made values with a real bcrypt hash of the bus password, as RootSecrets makes.
+func testCredentials(t *testing.T, store, bus string) RootCredentials {
+	t.Helper()
+	hash, err := bcrypt.GenerateFromPassword([]byte(bus), bcrypt.MinCost)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return RootCredentials{Store: store, Bus: bus, BusHash: string(hash)}
+}
+
+// theExample is the one genesis bundle, rewritten for an image as the installer does.
+func theExample(t *testing.T) Rewritten {
+	t.Helper()
 	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
 	if err != nil {
 		t.Skip("no example bundle beside this checkout")
@@ -20,81 +31,142 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"}
+	return r
+}
+
+// The produced bundle carries no well-known credential: the store reads its password from the
+// file genesis made, and the controller's account on the bus is the made one in both places the
+// bundle names it — the address the control plane dials and the user list the bus is raised with
+// (novox/hq issue 071, ADR 0331 §3).
+func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
+	r := theExample(t)
+	creds := testCredentials(t, "STORE-PW-40-characters-of-random-base64u", "BUS-PW-40-characters-of-random-base64uuu")
 	got, err := RewriteRoot(&r, creds)
 	if err != nil {
 		t.Fatal(err)
 	}
 	text := string(r.Bundle)
-	for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} {
+	for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "controller:bootstrap@"} {
 		if strings.Contains(text, gone) {
 			t.Errorf("the produced bundle still says %s", gone)
 		}
 	}
-	if got.StoreURLs < 3 || got.BrokerURLs < 2 {
-		t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs)
+	if got.StoreURLs < 3 || got.BusURLs != 1 {
+		t.Errorf("rewrote %d store and %d bus connections; the template has three and one", got.StoreURLs, got.BusURLs)
 	}
-	var store, action bool
+	var store, accounts, control bool
 	for _, res := range r.Declaration.Resources {
 		switch x := res.(type) {
 		case *declaration.Container:
-			if x.Name != "mesh-store" {
+			switch x.Name {
+			case "mesh-store":
+				store = true
+				if _, has := x.Env["POSTGRES_PASSWORD"]; has {
+					t.Error("the store still takes its password from its environment")
+				}
+				if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
+					t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
+				}
+				if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
+					t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
+				}
+			case r.TempName:
+				control = true
+				if want := "nats://controller:" + creds.Bus + "@127.0.0.1:5671"; x.Env[BusVar] != want {
+					t.Errorf("the temporary control plane dials the bus at %q, not with the made account", x.Env[BusVar])
+				}
+			}
+		case *declaration.File:
+			if x.ID != "bus-accounts" {
 				continue
 			}
-			store = true
-			if _, has := x.Env["POSTGRES_PASSWORD"]; has {
-				t.Error("the store still takes its password from its environment")
+			accounts = true
+			hash := hashOf(t, x.Content, "controller")
+			if bcrypt.CompareHashAndPassword([]byte(hash), []byte(creds.Bus)) != nil {
+				t.Errorf("the bus is raised with a controller account whose hash %q is not the made password's", hash)
 			}
-			if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
-				t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
-			}
-			if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
-				t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
-			}
-		case *declaration.Action:
-			if x.ID != "broker-admin" {
-				continue
-			}
-			action = true
-			if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") {
-				t.Errorf("the broker-admin action is %v in %q", x.Command, x.In)
+			if bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")) == nil {
+				t.Error("the bus is still raised with the controller at bootstrap")
 			}
 		}
 	}
-	if !store || !action {
-		t.Fatalf("store=%v action=%v", store, action)
-	}
-	// The order matters: the broker's password changes after it answers and before the control
-	// plane, which dials it with the new one, is raised.
-	var readyAt, adminAt, controlAt int
-	for i, res := range r.Declaration.Resources {
-		switch res.Identity() {
-		case "broker-ready":
-			readyAt = i
-		case "broker-admin":
-			adminAt = i
-		case "control-plane":
-			controlAt = i
-		}
-	}
-	if !(readyAt < adminAt && adminAt < controlAt) {
-		t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt)
+	if !store || !accounts || !control {
+		t.Fatalf("store=%v accounts=%v control=%v", store, accounts, control)
 	}
 }

-// A template that no longer says what this expects is refused, not half-rewritten.
-func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) {
+// hashOf is the password hash a NATS user list gives one user.
+func hashOf(t *testing.T, list, user string) string {
+	t.Helper()
+	marker := `{ user: "` + user + `", password: "`
+	at := strings.Index(list, marker)
+	if at < 0 {
+		t.Fatalf("the user list names no %s:\n%s", user, list)
+	}
+	rest := list[at+len(marker):]
+	return rest[:strings.Index(rest, `"`)]
+}
+
+// The bus's placeholders are what the installer looks for, and a template naming no bus connection
+// it knows is refused at preflight, naming the placeholder (novox/hq ADR 0331 §3, §4).
+func TestABundleNamingNoBusConnectionItKnowsIsRefusedByItsPlaceholder(t *testing.T) {
 	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
 	if err != nil {
 		t.Skip("no example bundle beside this checkout")
 	}
-	changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1)
-	r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32))
+	parse := func(b string) *declaration.Declaration {
+		d, err := declaration.ParseFileTrusted([]byte(b))
+		if err != nil {
+			t.Fatal(err)
+		}
+		return d
+	}
+	if err := BusPlaceholders(template, parse(string(template))); err != nil {
+		t.Fatalf("the shipped bundle is refused: %v", err)
+	}
+	for _, c := range []struct{ what, from, to, names string }{
+		{"no bus address", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`, "", BusVar},
+		{"the previous broker's", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671"`,
+			`"MESH_BUS_NATS": "amqp://guest:guest@127.0.0.1:5671"`, BusURLPlaceholder},
+		{"no pinned certificate", `"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"`, `"MESH_OTHER": "x"`, CertificateVar},
+		{"no controller in the user list", `{ user: \"controller\"`, `{ user: \"someone\"`, BusUserPlaceholder},
+	} {
+		if !strings.Contains(string(template), c.from) {
+			t.Fatalf("%s: the bundle does not say %s", c.what, c.from)
+		}
+		changed := strings.Replace(string(template), c.from, c.to, 1)
+		err := BusPlaceholders([]byte(changed), parse(changed))
+		if err == nil {
+			t.Errorf("%s: accepted", c.what)
+			continue
+		}
+		if !strings.Contains(err.Error(), c.names) {
+			t.Errorf("%s: refused without naming %s: %v", c.what, c.names, err)
+		}
+		if strings.Contains(err.Error(), "guest:guest") {
+			t.Errorf("%s: the refusal quotes a password: %v", c.what, err)
+		}
+	}
+}
+
+// The account is made once, with its hash, and a second run puts the same bytes in the bundle — a
+// hash salted afresh every run would rewrite the bus's user list every run.
+func TestTheBusAccountIsKeptWithItsHash(t *testing.T) {
+	dir := t.TempDir()
+	first, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil {
-		t.Fatal("a template with an unknown store password was rewritten")
+	if !first.BusMade || len(first.Bus) != 40 || bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) != nil {
+		t.Fatalf("first run: made=%v hash ok=%v", first.BusMade, bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) == nil)
+	}
+	second, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if second.BusMade || second.Bus != first.Bus || second.BusHash != first.BusHash {
+		t.Fatalf("a second run changed the bus account: made=%v same=%v same hash=%v",
+			second.BusMade, second.Bus == first.Bus, second.BusHash == first.BusHash)
 	}
 }

@@ -125,60 +197,25 @@ func TestRootSecretsAreKeptAcrossRuns(t *testing.T) {
 // Nothing the installer says after making the credentials contains them.
 func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) {
 	var said []string
-	say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"})
+	say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Bus: "BUS-PW"})
 	say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)")
-	say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)")
+	say("failed control-plane (MESH_BUS_NATS=nats://controller:BUS-PW@127.0.0.1:5671)")
 	for _, l := range said {
-		if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") {
+		if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BUS-PW") {
 			t.Errorf("said a credential: %s", l)
 		}
 	}
-	if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") {
+	if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "controller:…@") {
 		t.Errorf("the lines were not the same lines with the values masked: %v", said)
 	}
 }

-// The broker-admin marker is written with a line ending, because the verify reads it with `read`.
-func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) {
-	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
-	if err != nil {
-		t.Skip("no example bundle beside this checkout")
-	}
-	r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
-		t.Fatal(err)
-	}
-	for _, res := range r.Declaration.Resources {
-		a, ok := res.(*declaration.Action)
-		if !ok || a.ID != "broker-admin" {
-			continue
-		}
-		cmd := strings.Join(a.Command, " ")
-		if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") {
-			t.Errorf("the marker is written without a line ending: %s", cmd)
-		}
-		if !strings.Contains(strings.Join(a.Verify, " "), "read m <") {
-			t.Errorf("the verify does not read the marker: %v", a.Verify)
-		}
-	}
-}
-
 // The foundation filters before anything listens: nftables is in place before the store, and its
 // rules refuse the store's and broker's client ports from outside while keeping ssh, the bus a node
 // enrols over and the registry a node pulls from (novox/hq issue 054).
 func TestTheFoundationFiltersBeforeAnythingListens(t *testing.T) {
-	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
-	if err != nil {
-		t.Skip("no example bundle beside this checkout")
-	}
-	r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
+	r := theExample(t)
+	if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
 		t.Fatal(err)
 	}
 	var filterAt, loadedAt, storeAt, brokerAt = -1, -1, -1, -1