Merge pull request 'Genesis makes the root secrets and the operator key, and installs the vault' (#14) from feat/secrets-vault into main

internal/bootstrap/bootstrap.go

@@ -48,6 +48,7 @@ const (
 	StepApply        Step = "apply"
 	StepVerify       Step = "verify"
 	StepEnrol        Step = "enrol"
+	StepOperator     Step = "operator"
 	StepRegistry     Step = "registry"
 	StepPublish      Step = "publish"
 	StepControlPlane Step = "control-plane"
@@ -57,10 +58,13 @@ const (
 	StepSDK          Step = "sdk"
 	StepBase         Step = "base"
 	StepStore        Step = "store"
+	StepBroker       Step = "broker"
+	StepVault        Step = "vault"
 	StepCatalogue    Step = "catalogue"
 	StepNetwork      Step = "network"
 	StepFilter       Step = "filter"
 	StepExtras       Step = "extras"
+	StepExport       Step = "export"
 )
 
 // Steps in the order they happen, so a failure can say "step 2 of 11".
@@ -76,12 +80,12 @@ const (
 // mesh made, out of a repository and a commit it can name, and can therefore make again.
 var Steps = []Step{
 	StepPreflight, StepLoad, StepBuild, StepBundle, StepApply, StepVerify,
-	StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder,
+	StepEnrol, StepOperator, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder,
 	StepPackages, StepSDK,
 	// Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to
 	// build, to say what it holds, on its network, filtering. They used to be things somebody
 	// typed afterwards, which is how they went missing without anything complaining.
-	StepBase, StepStore, StepCatalogue, StepNetwork, StepFilter, StepExtras,
+	StepBase, StepStore, StepBroker, StepVault, StepCatalogue, StepNetwork, StepFilter, StepExtras, StepExport,
 }
 
 // Error is a failure, named by the step it happened in.
@@ -201,8 +205,13 @@ type Deps struct {
 
 // Result is what the bootstrap did, in the shape `--json` prints.
 type Result struct {
-	System string `json:"system"`
+	// OperatorKey is where the operator's private key was written; OperatorKeyMade whether this
-	DryRun bool   `json:"dry-run,omitempty"`
+	// run made it. RootExport is where the operator-sealed export landed.
+	OperatorKey     string `json:"operator-key,omitempty"`
+	OperatorKeyMade bool   `json:"operator-key-made,omitempty"`
+	RootExport      string `json:"root-export,omitempty"`
+	System          string `json:"system"`
+	DryRun          bool   `json:"dry-run,omitempty"`
 
 	// Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and
 	// what the produced bundle names it by.
@@ -380,6 +389,36 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 	}
 	result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out
 	result.Temporary = rewritten.TempName
+	// The mesh's root credentials: made here, never the template's (novox/hq issue 071).
+	creds, err := RootSecrets(o.DryRun)
+	if err != nil {
+		return result, failed(StepBundle, err)
+	}
+	if err := RefuseExistingServers(ctx, d.Run, creds); err != nil {
+		return result, failed(StepBundle, err)
+	}
+	// From here on nothing this installer says contains the values it just made.
+	say = Masking(say, creds)
+	root, err := RewriteRoot(&rewritten, creds)
+	if err != nil {
+		return result, failed(StepBundle, err)
+	}
+	for _, c := range []struct {
+		what, path string
+		made       bool
+	}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} {
+		switch {
+		case c.made && o.DryRun:
+			say(fmt.Sprintf("  %-17s would be made and kept at %s (0600)", c.what, c.path))
+		case c.made:
+			say(fmt.Sprintf("  %-17s made, kept at %s (0600)", c.what, c.path))
+		default:
+			say(fmt.Sprintf("  %-17s already at %s — kept", c.what, c.path))
+		}
+	}
+	say(fmt.Sprintf("  credentials       the template's bootstrap and guest are gone: %d store and %d broker "+
+		"connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers",
+		root.StoreURLs, root.BrokerURLs))
 	if rewritten.Renamed {
 		say(fmt.Sprintf("  control plane     %s, renamed from %s",
 			rewritten.TempName, rewritten.WasCalled))
@@ -509,6 +548,18 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 		return result, failed(StepEnrol, err)
 	}
 
+	// ---- operator — the key the mesh's root secrets are also sealed to, held by a person ------
+	//
+	// Before anything is accepted into the mesh: the store's and broker's credentials go in during
+	// the control-plane step, and they must be sealed to this key as well as to the node, or they
+	// are as unrecoverable as the constants they replaced (novox/hq ADR 0085, amended).
+	say("operator — a key the mesh seals its root secrets to, held by a person and never by the mesh")
+	operator, err := MakeOperatorKey(ctx, o, temporary, say)
+	result.OperatorKey, result.OperatorKeyMade = operator.Path, operator.Made
+	if err != nil {
+		return result, failed(StepOperator, err)
+	}
+
 	// ---- 7. registry ----------------------------------------------------------------------
 	say("registry — somewhere for this mesh to keep its own images")
 	registry, err := InstallRegistry(ctx, o, d, temporary, say)
@@ -598,6 +649,20 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 		return result, failed(StepStore, err)
 	}
 
+	// ---- broker ---------------------------------------------------------------------------
+	say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two")
+	if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil {
+		return result, failed(StepBroker, err)
+	}
+
+	// ---- vault ----------------------------------------------------------------------------
+	// A foundation module (novox/hq ADR 0085, amended): it keeps every operator-sealed secret on
+	// its own disk, outside the store, from the first push that carries one.
+	say("vault — the mesh's secrets get an owner, and their operator-sealed copies a place to live")
+	if err := InstallVault(ctx, o, permanentControl, say); err != nil {
+		return result, failed(StepVault, err)
+	}
+
 	// ---- 15. catalogue --------------------------------------------------------------------
 	say("catalogue — the module graph: what is held, what a change reaches, what to rebuild")
 	if err := InstallFromCatalogue(ctx, o, permanentControl, "mesh-catalog", say); err != nil {
@@ -622,6 +687,14 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 		return result, failed(StepExtras, err)
 	}
 
+	// ---- export — what the operator keeps beside the key ---------------------------------
+	say("export — every root secret, sealed to the operator key, written beside it")
+	exported, err := ExportRootSecrets(ctx, o, permanentControl, say)
+	result.RootExport = exported
+	if err != nil {
+		return result, failed(StepExport, err)
+	}
+
 	say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " +
 		"sits on its private network, and filters what modules declared.")
 	say("what remains is somebody else's: adding nodes, and assigning what they should run.")

internal/bootstrap/control.go

@@ -274,7 +274,7 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes
 		// installer has neither a terminal to be prompted at nor a way to write to a command's
 		// standard input through the runner every applier in this repository shares.
 		at := "/accepting-" + secret
-		if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
+		if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
 			return delivered, err
 		}
 		if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret,

internal/bootstrap/operator.go

@@ -0,0 +1,100 @@
+package bootstrap
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/novox/mesh-host/internal/identity"
+)
+
+// The operator's sealing key: made at genesis, before the mesh is told any secret.
+//
+// Every secret a module holds for itself is sealed to the node that uses it; from here on it is
+// sealed to this key as well, and the vault keeps those copies (novox/hq ADR 0085, amended). The
+// private half is written once, beside the produced bundle, and given to nothing: the mesh
+// records the public half and can open nothing it seals to it. The operator copies the file off
+// the machine and keeps it — it is what recovers the mesh's root secrets when a node cannot.
+//
+// **Before enrolment's first `secret accept`**, or the credentials genesis made would be sealed
+// to the node alone and be exactly as unrecoverable as the constants they replaced.
+
+// OperatorKeyFile is where the private half is written, beside the bundle.
+func OperatorKeyFile(o Options) string {
+	return filepath.Join(filepath.Dir(o.Out), "operator.key")
+}
+
+// RootExportFile is where the export of every operator-sealed secret is written at the end.
+func RootExportFile(o Options) string {
+	return filepath.Join(filepath.Dir(o.Out), "root-secrets.export.json")
+}
+
+type OperatorKey struct {
+	Path        string
+	Fingerprint string
+	Made        bool
+}
+
+// MakeOperatorKey makes the key if this machine has none, and tells the mesh its public half.
+func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say func(string)) (OperatorKey, error) {
+	out := OperatorKey{Path: OperatorKeyFile(o)}
+	var key identity.SealingKey
+	if _, err := os.Stat(out.Path); err == nil {
+		key, err = identity.LoadSealingKey(out.Path)
+		if err != nil {
+			return out, err
+		}
+		say("  operator key      already at " + out.Path + " — kept")
+	} else if os.IsNotExist(err) {
+		key, err = identity.GenerateSealingKey()
+		if err != nil {
+			return out, err
+		}
+		if err := os.MkdirAll(filepath.Dir(out.Path), 0o755); err != nil {
+			return out, err
+		}
+		if err := os.WriteFile(out.Path, []byte(key.Private+"\n"), 0o600); err != nil {
+			return out, err
+		}
+		out.Made = true
+	} else {
+		return out, err
+	}
+	sum := sha256.Sum256([]byte(key.Public))
+	out.Fingerprint = "sha256:" + hex.EncodeToString(sum[:8])
+
+	if _, err := control.tell(ctx, "operator", "key", "set", key.Public); err != nil {
+		return out, err
+	}
+	if out.Made {
+		say("  operator key      " + out.Fingerprint + " — private half at " + out.Path + " (0600)")
+		say("                    COPY IT OFF THIS MACHINE AND KEEP IT: it opens the mesh's root secrets, and")
+		say("                    nothing else does. The mesh holds only the public half.")
+	} else {
+		say("  operator key      " + out.Fingerprint + " — the mesh seals its root secrets to it")
+	}
+	return out, nil
+}
+
+// ExportRootSecrets writes the export beside the operator key: every secret sealed to it, as
+// ciphertext, and the honest list of what is not. What the vault keeps on its disk, kept once
+// more by the person who holds the key.
+func ExportRootSecrets(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
+	path := RootExportFile(o)
+	body, err := control.tell(ctx, "secret", "export")
+	if err != nil {
+		return path, err
+	}
+	if !strings.Contains(body, `"kept"`) {
+		return path, fmt.Errorf("`secret export` did not produce an export:\n%s", body)
+	}
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		return path, err
+	}
+	say("  exported          " + path + " (0600) — ciphertext, sealed to the operator key; keep it with the key")
+	return path, nil
+}

internal/bootstrap/phase3.go

@@ -72,6 +72,27 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
 	}
 	say("  adopting          " + store.Name + " — the store the foundation raised, unchanged")
 
+	// The superuser is the foundation's, made at genesis — carried in before the push, or the push
+	// would seal random bytes where a working password has to be and the provisioner would not open
+	// the store it is meant to manage.
+	if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)",
+		func() error {
+			return deliverCredential(ctx, o, control, module, "superuser", StoreSuperuserFile, "the store's superuser", say)
+		},
+		say); err != nil {
+		return err
+	}
+	say("  adopted           mesh-store — the foundation's store is now the " + module + " module")
+	return nil
+}
+
+// installProvider registers, builds, issues, assigns and pushes one catalogue module, with one
+// thing done just before the push — the moment a credential the mesh could not have made has to
+// be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker
+// and the vault each need it or need the shape, and three copies of it drifted.
+func installProvider(ctx context.Context, o Options, control controlPlane, module string,
+	manifest []byte, buildNote string, beforePush func() error, say func(string)) error {
+
 	remote := "/" + module + "-module.json"
 	if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
 		return err
@@ -86,7 +107,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
 			"the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where a builder "+
 			"clones it", module)
 	}
-	say("  building          " + module + " (the provisioner; the server is adopted, not built)")
+	say(strings.TrimRight("  building          "+module+" "+buildNote, " "))
 	if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository,
 		"--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil {
 		return err
@@ -101,18 +122,80 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
 	if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
 		return err
 	}
+	if beforePush != nil {
+		if err := beforePush(); err != nil {
+			return err
+		}
+	}
+	_, err := pushNode(ctx, o, control, say)
+	return err
+}
 
-	// The superuser is the foundation's, made at genesis — carried in before the push, or the push
+// deliverCredential carries a credential genesis made into a module as its own secret, through
-	// would seal random bytes where a working password has to be and the provisioner would not open
+// `secret accept`: the mesh cannot invent the value a running server already has.
-	// the store it is meant to manage.
+func deliverCredential(ctx context.Context, o Options, control controlPlane, module, secret, file, what string,
-	if err := deliverSuperuser(ctx, o, control, module, store, say); err != nil {
+	say func(string)) error {
+
+	value, err := readCredentialFile(file)
+	if err != nil {
+		return fmt.Errorf("%s is not at %s, so the %s module has nothing to open its server with — "+
+			"and the mesh cannot invent the one the server already has: %w", what, file, module, err)
+	}
+	at := "/accepting-" + secret
+	if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
 		return err
 	}
-
+	if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil {
-	if _, err := pushNode(ctx, o, control, say); err != nil {
 		return err
 	}
-	say("  adopted           mesh-store — the foundation's store is now the " + module + " module")
+	say("  accepted          " + secret + " — " + what + ", as genesis made it")
+	return nil
+}
+
+// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same
+// shape as InstallStore, for the same reasons. The administrator's password is the one genesis
+// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can
+// reach the management API as it.
+func InstallBroker(ctx context.Context, o Options, control controlPlane,
+	foundation *declaration.Declaration, say func(string)) error {
+
+	const module = "lavinmq"
+	manifest, err := readManifest(o.Catalogue, module)
+	if err != nil {
+		return err
+	}
+	broker, err := brokerIn(foundation)
+	if err != nil {
+		return err
+	}
+	if err := serverMatchesFoundation(manifest, broker, module); err != nil {
+		return err
+	}
+	say("  adopting          " + broker.Name + " — the broker the foundation raised, unchanged")
+	if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)",
+		func() error {
+			return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say)
+		},
+		say); err != nil {
+		return err
+	}
+	say("  adopted           " + broker.Name + " — the foundation's broker is now the " + module + " module")
+	return nil
+}
+
+// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
+// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider,
+// and from its first push it keeps the export of every operator-sealed secret on its own disk.
+func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error {
+	const module = "mesh-vault"
+	manifest, err := readManifest(o.Catalogue, module)
+	if err != nil {
+		return err
+	}
+	if err := installProvider(ctx, o, control, module, manifest, "", nil, say); err != nil {
+		return err
+	}
+	say("  installed         " + module + " — the operator-sealed export now lives on this machine, outside the store")
 	return nil
 }
 
@@ -176,30 +259,3 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu
 			"store with. Its server container has to carry the name the foundation raised",
 		module, store.Name)
 }
-
-// deliverSuperuser carries the store's superuser password into the module.
-//
-// It is the foundation's, set on the bundle's store container at genesis; the mesh cannot invent a
-// credential that already made the databases, so it goes in through `secret accept`, exactly as the
-// control plane's store connections do (control.go deliverStores).
-func deliverSuperuser(ctx context.Context, o Options, control controlPlane, module string,
-	store *declaration.Container, say func(string)) error {
-
-	const secret = "superuser"
-	value := strings.TrimSpace(store.Env["POSTGRES_PASSWORD"])
-	if value == "" {
-		return fmt.Errorf(
-			"the foundation's store names no POSTGRES_PASSWORD, so the %s module has no superuser "+
-				"to open it with — and the mesh cannot invent the one that already made the databases",
-			module)
-	}
-	at := "/accepting-" + secret
-	if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
-		return err
-	}
-	if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil {
-		return err
-	}
-	say("  accepted          " + secret + " — the store's superuser, as the foundation made it")
-	return nil
-}

internal/bootstrap/phase_packages.go

@@ -270,7 +270,7 @@ func packagePasswords() (db, admin, builder string, err error) {
 func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string,
 	say func(string)) error {
 	at := "/accepting-npm-password"
-	if err := control.carrying(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil {
+	if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil {
 		return err
 	}
 	if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil {

internal/bootstrap/rewrite.go

@@ -316,21 +316,26 @@ func sortStrings(values []string) {
 // writeBundleFile puts the produced bundle where a person can read it, creating the directory it
 // lives in.
 //
-// 0644, and that is deliberate: this file names an image and describes a foundation, and it holds
+// 0600: the produced bundle carries the credentials genesis made — the store's and the broker's,
-// the bootstrap credentials the template happens to carry — which are the same ones anybody can
+// inside the temporary control plane's connection strings (novox/hq issue 071). It used to be
-// read in the template itself. It is meant to be read. What must not be world-readable is the
+// 0644 and say so was fine because the template's credentials were the same ones anybody could
-// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`).
+// read in the template; they are not any more. Still meant to be read, by root.
 func writeBundleFile(path string, content []byte) error {
 	if dir := filepath.Dir(path); dir != "" && dir != "." {
 		if err := os.MkdirAll(dir, 0o755); err != nil {
 			return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err)
 		}
 	}
-	if err := os.WriteFile(path, content, 0o644); err != nil {
+	if err := os.WriteFile(path, content, 0o600); err != nil {
 		return fmt.Errorf(
 			"cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+
 				"applying something nobody can read afterwards is how a machine becomes a mystery",
 			path, err)
 	}
+	// The mode above applies only when the file is created. A bundle an earlier installer left at
+	// 0644 would keep that while now carrying real credentials, with this function saying 0600.
+	if err := os.Chmod(path, 0o600); err != nil {
+		return fmt.Errorf("cannot make the produced bundle %s readable by root alone: %w", path, err)
+	}
 	return nil
 }

internal/bootstrap/rootsecrets.go

@@ -0,0 +1,269 @@
+package bootstrap
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// The mesh's root credentials, made at genesis rather than copied from the template.
+//
+// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq
+// issue 071). The store's superuser and the broker's administrator are the two credentials every
+// other one rests on, and they were the two that were not secret: constants in a file anybody can
+// read, carried into the mesh by `secret accept` and marked as something the mesh must never
+// replace — which is correct for a credential that already created the databases, and made the
+// well-known value permanent.
+//
+// So the installer makes them. Two random values, **made once and kept on this machine** at the
+// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three
+// adopts the store and the broker, `secret accept` carries in exactly the value the servers were
+// raised with, and the host's later write of the sealed secret lands the same bytes in the same
+// file. A second run finds the files and changes nothing, which is what lets the installer say
+// "already done" about a store it must not restart.
+//
+// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a
+// container's environment is in `docker inspect` for ever; the module that adopts the store
+// declares the same file mount, so the two specs are one and the applier reconciles rather than
+// recreates (phase3.go). The broker has no such file: its image's default administrator is changed
+// in place by an action once the broker answers, and the produced bundle carries that action.
+const (
+	// StoreSuperuserFile is where the store's superuser password lives on the machine — the
+	// postgres module's own-secret path, so genesis and adoption write the same file.
+	StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
+	// BrokerAdminFile is the same for the broker's administrator — the lavinmq module's.
+	BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret"
+	// BrokerAdminUser is the broker's administrator. The image's default account, kept by name
+	// and given a password that is not the image's default; a renamed account would have to be
+	// created before anything can authenticate, and the thing that creates accounts is the thing
+	// that has to authenticate first.
+	BrokerAdminUser = "guest"
+
+	storeSuperuserMount = "/run/secrets/superuser"
+
+	// What the template says, matched exactly. A template that says something else is a template
+	// this installer does not know how to make safe, and it says so rather than guessing.
+	templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
+	templateStoreVolumes  = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]`
+	templateStoreURL      = "postgres:bootstrap@"
+	templateBrokerURL     = "guest:guest@"
+	templateBrokerReady   = "\"verify\": [\"lavinmqctl\", \"status\"]\n    },"
+	brokerAdminMarker     = "/var/lib/lavinmq/.mesh-admin"
+)
+
+// RootCredentials are the two values, and whether this run made them.
+type RootCredentials struct {
+	Store, Broker         string
+	StoreMade, BrokerMade bool
+}
+
+// RootSecrets reads the credentials this machine already holds, or makes them.
+//
+// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the
+// real one, and a machine that was only asked is not left holding half a genesis.
+func RootSecrets(dryRun bool) (RootCredentials, error) {
+	var out RootCredentials
+	var err error
+	if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil {
+		return out, err
+	}
+	if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil {
+		return out, err
+	}
+	return out, nil
+}
+
+func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
+	value, err = readCredentialFile(path)
+	if err == nil {
+		return value, false, nil
+	}
+	if !os.IsNotExist(err) {
+		return "", false, err
+	}
+	value, err = freshSecret()
+	if err != nil {
+		return "", false, err
+	}
+	if dryRun {
+		return value, true, nil
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return "", false, err
+	}
+	// Written whole and renamed into place, at 0600, owned by whoever runs the installer — root,
+	// which is also who the host runs as when it later writes the sealed copy here.
+	tmp := path + ".genesis"
+	if err := os.WriteFile(tmp, []byte(value+"\n"), 0o600); err != nil {
+		return "", false, err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		return "", false, err
+	}
+	return value, true, nil
+}
+
+// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
+// Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable".
+func readCredentialFile(path string) (string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	value := strings.TrimRight(string(raw), "\r\n")
+	if value == "" {
+		return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path)
+	}
+	return value, nil
+}
+
+// credentialFingerprint names a credential without being one — what the broker-admin action
+// leaves on the broker's volume, so its verify holds for this value and not for any value.
+func credentialFingerprint(value string) string {
+	sum := sha256.Sum256([]byte(value))
+	return hex.EncodeToString(sum[:8])
+}
+
+// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40
+// characters, URL-safe — it lands inside connection strings.
+func freshSecret() (string, error) {
+	b := make([]byte, 30)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(b), nil
+}
+
+// Masking makes a reporter that never says the credentials this run made.
+//
+// The applier reports each action with its command line, and two of them now carry a real
+// password — the context schemas' connection strings and the broker's change_password. Those
+// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The
+// exact values are known here, so they are replaced wherever they appear, in every line said.
+func Masking(say func(string), c RootCredentials) func(string) {
+	replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…")
+	if c.Store == "" || c.Broker == "" {
+		return say
+	}
+	return func(line string) { say(replacer.Replace(line)) }
+}
+
+// RefuseExistingServers stops a run that would put a made credential in front of a server raised
+// by an earlier installer with the template's.
+//
+// The store's password is set by initdb, once, on an empty volume; the broker's by the action
+// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no
+// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a
+// bundle that dials with passwords the servers do not have — failing three steps later, in the
+// schemas, with nothing pointing back here. Refused by name instead, with the way forward.
+func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error {
+	for _, check := range []struct {
+		made   bool
+		volume string
+		what   string
+		file   string
+	}{
+		{c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile},
+		{c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile},
+	} {
+		if !check.made {
+			continue
+		}
+		if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil {
+			continue // no such volume: a fresh machine, which is the case this installer makes
+		}
+		return fmt.Errorf(
+			"this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+
+				"by an earlier installer with the template's password. A new one made here would not open it. "+
+				"Put the password the %s has into %s (0600, the value alone) and run again; then change it "+
+				"on the server and accept the new value — this installer does not rotate a running %s",
+			check.what, check.volume, check.file, check.what, check.file, check.what)
+	}
+	return nil
+}
+
+// RootRewrite says what RewriteRoot did to the bundle.
+type RootRewrite struct {
+	StoreURLs, BrokerURLs int
+}
+
+// RewriteRoot puts the made credentials into the produced bundle, in place of the template's.
+//
+// Byte for byte, like the image rewrite, so the file keeps its comments and a person can read what
+// was applied. Every replacement is counted and a count of zero is refused: a template that no
+// longer says what this expects is one whose credentials this would silently leave at the
+// well-known values, which is the fault this exists to remove.
+func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
+	var out RootRewrite
+	bundle := r.Bundle
+
+	// The store: a file, not an environment variable.
+	var err error
+	if bundle, err = replaceOnce(bundle, templateStorePassword,
+		`"POSTGRES_PASSWORD_FILE": "`+storeSuperuserMount+`"`, "the store's password"); err != nil {
+		return out, err
+	}
+	if bundle, err = replaceOnce(bundle, templateStoreVolumes,
+		`"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`,
+		"the store's volumes"); err != nil {
+		return out, err
+	}
+	// Everything that dials the store or the broker with the template's credentials.
+	out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL))
+	if out.StoreURLs == 0 {
+		return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL)
+	}
+	bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@"))
+	out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL))
+	if out.BrokerURLs == 0 {
+		return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL)
+	}
+	bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@"))
+
+	// The broker's administrator, changed once the broker answers and before anything dials it.
+	// Verified by a marker on the broker's own data volume holding this password's fingerprint —
+	// the image carries nothing that can try a password from inside, and a marker that merely
+	// existed would let a regenerated password go unapplied for ever. What proves the password
+	// works is the control plane answering over it, a few resources later. The marker ends in a
+	// newline because the verify reads it with the shell's `read`, which fails at end of file
+	// without one — an action that ran and a verify that said no, once, in the lab.
+	fp := credentialFingerprint(c.Broker)
+	action := templateBrokerReady + "\n" +
+		"    {\n" +
+		"      \"id\": \"broker-admin\",\n" +
+		"      \"type\": \"action\",\n" +
+		"      \"in\": \"mesh-broker\",\n" +
+		"      \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" +
+		"      \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" +
+		"    },"
+	if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil {
+		return out, err
+	}
+
+	parsed, err := declaration.ParseFileTrusted(bundle)
+	if err != nil {
+		return out, fmt.Errorf("the bundle stopped being a declaration after its credentials were rewritten, which is this installer's fault: %w", err)
+	}
+	r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources)
+	return out, nil
+}
+
+func replaceOnce(in []byte, from, to, what string) ([]byte, error) {
+	switch n := bytes.Count(in, []byte(from)); n {
+	case 1:
+		return bytes.Replace(in, []byte(from), []byte(to), 1), nil
+	case 0:
+		return nil, fmt.Errorf("the template does not say %s the way this installer expects (%s), so it cannot be made safe here", what, from)
+	default:
+		return nil, fmt.Errorf("the template says %s %d times, and this installer expected once", what, n)
+	}
+}

internal/bootstrap/rootsecrets_test.go

@@ -0,0 +1,167 @@
+package bootstrap
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// The produced bundle carries no well-known credential: the store reads its password from the
+// file genesis made, every connection string names the made values, and the broker's default
+// administrator is changed by an action before anything dials it (novox/hq issue 071).
+func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
+	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
+	if err != nil {
+		t.Skip("no example bundle beside this checkout")
+	}
+	r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
+	if err != nil {
+		t.Fatal(err)
+	}
+	creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"}
+	got, err := RewriteRoot(&r, creds)
+	if err != nil {
+		t.Fatal(err)
+	}
+	text := string(r.Bundle)
+	for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} {
+		if strings.Contains(text, gone) {
+			t.Errorf("the produced bundle still says %s", gone)
+		}
+	}
+	if got.StoreURLs < 3 || got.BrokerURLs < 2 {
+		t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs)
+	}
+	var store, action bool
+	for _, res := range r.Declaration.Resources {
+		switch x := res.(type) {
+		case *declaration.Container:
+			if x.Name != "mesh-store" {
+				continue
+			}
+			store = true
+			if _, has := x.Env["POSTGRES_PASSWORD"]; has {
+				t.Error("the store still takes its password from its environment")
+			}
+			if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
+				t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
+			}
+			if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
+				t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
+			}
+		case *declaration.Action:
+			if x.ID != "broker-admin" {
+				continue
+			}
+			action = true
+			if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") {
+				t.Errorf("the broker-admin action is %v in %q", x.Command, x.In)
+			}
+		}
+	}
+	if !store || !action {
+		t.Fatalf("store=%v action=%v", store, action)
+	}
+	// The order matters: the broker's password changes after it answers and before the control
+	// plane, which dials it with the new one, is raised.
+	var readyAt, adminAt, controlAt int
+	for i, res := range r.Declaration.Resources {
+		switch res.Identity() {
+		case "broker-ready":
+			readyAt = i
+		case "broker-admin":
+			adminAt = i
+		case "control-plane":
+			controlAt = i
+		}
+	}
+	if !(readyAt < adminAt && adminAt < controlAt) {
+		t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt)
+	}
+}
+
+// A template that no longer says what this expects is refused, not half-rewritten.
+func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) {
+	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
+	if err != nil {
+		t.Skip("no example bundle beside this checkout")
+	}
+	changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1)
+	r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil {
+		t.Fatal("a template with an unknown store password was rewritten")
+	}
+}
+
+// Made once and kept: a second run reads the same value; a dry run writes nothing.
+func TestRootSecretsAreKeptAcrossRuns(t *testing.T) {
+	dir := t.TempDir()
+	path := dir + "/superuser.secret"
+	first, made, err := keptOrMade(path, false)
+	if err != nil || !made || len(first) != 40 {
+		t.Fatalf("first: %q made=%v err=%v", first, made, err)
+	}
+	if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
+		t.Errorf("mode %v", info.Mode().Perm())
+	}
+	second, made, err := keptOrMade(path, false)
+	if err != nil || made || second != first {
+		t.Fatalf("second: %q made=%v err=%v", second, made, err)
+	}
+	dry := dir + "/dry.secret"
+	if _, made, err := keptOrMade(dry, true); err != nil || !made {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(dry); err == nil {
+		t.Fatal("a dry run wrote a secret")
+	}
+}
+
+// Nothing the installer says after making the credentials contains them.
+func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) {
+	var said []string
+	say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"})
+	say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)")
+	say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)")
+	for _, l := range said {
+		if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") {
+			t.Errorf("said a credential: %s", l)
+		}
+	}
+	if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") {
+		t.Errorf("the lines were not the same lines with the values masked: %v", said)
+	}
+}
+
+// The broker-admin marker is written with a line ending, because the verify reads it with `read`.
+func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) {
+	template, err := os.ReadFile("../../examples/foundation-first-node.lock")
+	if err != nil {
+		t.Skip("no example bundle beside this checkout")
+	}
+	r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
+		t.Fatal(err)
+	}
+	for _, res := range r.Declaration.Resources {
+		a, ok := res.(*declaration.Action)
+		if !ok || a.ID != "broker-admin" {
+			continue
+		}
+		cmd := strings.Join(a.Command, " ")
+		if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") {
+			t.Errorf("the marker is written without a line ending: %s", cmd)
+		}
+		if !strings.Contains(strings.Join(a.Verify, " "), "read m <") {
+			t.Errorf("the verify does not read the marker: %v", a.Verify)
+		}
+	}
+}

internal/bootstrap/talk.go

@@ -82,11 +82,9 @@ func (c controlPlane) carry(ctx context.Context, local, remote string) error {
 // landed unreadable, `secret accept` failed with `permission denied`, and what depended on it
 // crash-looped on material it never received. There is no shell in the image to chown it with.
 //
-// What goes through here is a module manifest and a store connection string. The connection is the
+// What goes through here is a module manifest — public, the same bytes as in the catalogue. A
-// same value the produced bundle already holds in the clear — a foundation names its own bootstrap
+// value that is secret goes through carryingSecret below. The file on the machine is removed at
-// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The
+// once, and the copy inside the container goes when the container does.
-// file on the machine is removed at once, and the copy inside the container goes when the
-// container does, which for the temporary control plane is step 10.
 func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error {
 	local := filepath.Join(os.TempDir(), name)
 	if err := os.WriteFile(local, content, 0o644); err != nil {
@@ -96,6 +94,38 @@ func (c controlPlane) carrying(ctx context.Context, name string, content []byte,
 	return c.carry(ctx, local, remote)
 }
 
+// controlPlaneUID is the account the control plane's image runs as — `USER 65534:65534` in its
+// Dockerfile — and so the only account inside the container that needs to read what is carried in.
+const controlPlaneUID = 65534
+
+// carryingSecret is carrying for a value that is a secret: staged in a directory only root can
+// enter, at 0600, owned by the control plane's own account — which `docker cp` keeps, so inside
+// the container the file is readable by the process that must read it and by nobody else. Since
+// genesis makes the mesh's root credentials rather than copying the template's (rootsecrets.go),
+// a store connection string or a broker password carried this way is a real secret, and 0644 in a
+// shared temporary directory would hand it to any local user for the length of the copy.
+func (c controlPlane) carryingSecret(ctx context.Context, name string, content []byte, remote string) error {
+	dir, err := os.MkdirTemp("", "mesh-carrying-")
+	if err != nil {
+		return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err)
+	}
+	defer os.RemoveAll(dir)
+	local := filepath.Join(dir, name)
+	if err := os.WriteFile(local, content, 0o600); err != nil {
+		return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err)
+	}
+	if err := os.Chown(local, controlPlaneUID, controlPlaneUID); err != nil {
+		// Not root — a test, or an installer run as a user, which no real genesis is. The
+		// directory is 0700, so nobody else on the machine can reach the file either way; inside
+		// the container the only account is the control plane's, so 0644 there is read by it and
+		// by nothing else. The narrower ownership is taken whenever it can be.
+		if err := os.Chmod(local, 0o644); err != nil {
+			return err
+		}
+	}
+	return c.carry(ctx, local, remote)
+}
+
 func indent(s string) string {
 	if s == "" {
 		return ""