From ee0c8b856e116e72173337f5032e0bb61f8ad954 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 00:12:55 +0200 Subject: [PATCH 1/4] Genesis makes the root secrets, the operator key, and installs the vault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The template raises the store with the password 'bootstrap' and the broker with its image's default administrator, and the installer carried both into the mesh as accepted secrets — permanent, and not secret (novox/hq issue 071). Now the installer makes both credentials, once, at the paths the postgres and lavinmq modules declare as their own secrets, rewrites the produced bundle to use them (the store reads its password from a file; the broker's default account is given the new password by an action before anything dials it), and writes the bundle at 0600 since it now carries them. Before the first secret is accepted it makes the operator's sealing key beside the bundle and gives the mesh the public half, so everything minted from there is sealed to it too (ADR 0085, amended). Phase three adopts the broker as the lavinmq module beside the store and installs mesh-vault as a foundation module; the run ends by writing the operator-sealed export beside the key. --- internal/bootstrap/bootstrap.go | 75 +++++++++- internal/bootstrap/operator.go | 107 ++++++++++++++ internal/bootstrap/phase3.go | 131 +++++++++++++++- internal/bootstrap/rewrite.go | 10 +- internal/bootstrap/rootsecrets.go | 197 +++++++++++++++++++++++++ internal/bootstrap/rootsecrets_test.go | 123 +++++++++++++++ 6 files changed, 630 insertions(+), 13 deletions(-) create mode 100644 internal/bootstrap/operator.go create mode 100644 internal/bootstrap/rootsecrets.go create mode 100644 internal/bootstrap/rootsecrets_test.go diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a640b5f..278f0ae 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -48,6 +48,7 @@ const ( StepApply Step = "apply" StepVerify Step = "verify" StepEnrol Step = "enrol" + StepOperator Step = "operator" StepRegistry Step = "registry" StepPublish Step = "publish" StepControlPlane Step = "control-plane" @@ -57,6 +58,8 @@ const ( StepSDK Step = "sdk" StepBase Step = "base" StepStore Step = "store" + StepBroker Step = "broker" + StepVault Step = "vault" StepCatalogue Step = "catalogue" StepNetwork Step = "network" StepFilter Step = "filter" @@ -76,12 +79,12 @@ const ( // mesh made, out of a repository and a commit it can name, and can therefore make again. var Steps = []Step{ StepPreflight, StepLoad, StepBuild, StepBundle, StepApply, StepVerify, - StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder, + StepEnrol, StepOperator, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder, StepPackages, StepSDK, // Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to // build, to say what it holds, on its network, filtering. They used to be things somebody // typed afterwards, which is how they went missing without anything complaining. - StepBase, StepStore, StepCatalogue, StepNetwork, StepFilter, StepExtras, + StepBase, StepStore, StepBroker, StepVault, StepCatalogue, StepNetwork, StepFilter, StepExtras, } // Error is a failure, named by the step it happened in. @@ -201,8 +204,13 @@ type Deps struct { // Result is what the bootstrap did, in the shape `--json` prints. type Result struct { - System string `json:"system"` - DryRun bool `json:"dry-run,omitempty"` + // OperatorKey is where the operator's private key was written; OperatorKeyMade whether this + // run made it. RootExport is where the operator-sealed export landed. + OperatorKey string + OperatorKeyMade bool + RootExport string + System string `json:"system"` + DryRun bool `json:"dry-run,omitempty"` // Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and // what the produced bundle names it by. @@ -380,6 +388,31 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro } result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out result.Temporary = rewritten.TempName + // The mesh's root credentials: made here, never the template's (novox/hq issue 071). + creds, err := RootSecrets(o.DryRun) + if err != nil { + return result, failed(StepBundle, err) + } + root, err := RewriteRoot(&rewritten, creds) + if err != nil { + return result, failed(StepBundle, err) + } + for _, c := range []struct { + what, path string + made bool + }{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} { + switch { + case c.made && o.DryRun: + say(fmt.Sprintf(" %-17s would be made and kept at %s (0600)", c.what, c.path)) + case c.made: + say(fmt.Sprintf(" %-17s made, kept at %s (0600)", c.what, c.path)) + default: + say(fmt.Sprintf(" %-17s already at %s — kept", c.what, c.path)) + } + } + say(fmt.Sprintf(" credentials the template's bootstrap and guest are gone: %d store and %d broker "+ + "connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers", + root.StoreURLs, root.BrokerURLs)) if rewritten.Renamed { say(fmt.Sprintf(" control plane %s, renamed from %s", rewritten.TempName, rewritten.WasCalled)) @@ -509,6 +542,18 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepEnrol, err) } + // ---- operator — the key the mesh's root secrets are also sealed to, held by a person ------ + // + // Before anything is accepted into the mesh: the store's and broker's credentials go in during + // the control-plane step, and they must be sealed to this key as well as to the node, or they + // are as unrecoverable as the constants they replaced (novox/hq ADR 0085, amended). + say("operator — a key the mesh seals its root secrets to, held by a person and never by the mesh") + operator, err := MakeOperatorKey(ctx, o, temporary, say) + result.OperatorKey, result.OperatorKeyMade = operator.Path, operator.Made + if err != nil { + return result, failed(StepOperator, err) + } + // ---- 7. registry ---------------------------------------------------------------------- say("registry — somewhere for this mesh to keep its own images") registry, err := InstallRegistry(ctx, o, d, temporary, say) @@ -598,6 +643,20 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepStore, err) } + // ---- 14b. broker ---------------------------------------------------------------------- + say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two") + if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { + return result, failed(StepBroker, err) + } + + // ---- 14c. vault ----------------------------------------------------------------------- + // A foundation module (novox/hq ADR 0085, amended): it keeps every operator-sealed secret on + // its own disk, outside the store, from the first push that carries one. + say("vault — the mesh's secrets get an owner, and their operator-sealed copies a place to live") + if err := InstallVault(ctx, o, permanentControl, say); err != nil { + return result, failed(StepVault, err) + } + // ---- 15. catalogue -------------------------------------------------------------------- say("catalogue — the module graph: what is held, what a change reaches, what to rebuild") if err := InstallFromCatalogue(ctx, o, permanentControl, "mesh-catalog", say); err != nil { @@ -622,6 +681,14 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepExtras, err) } + // ---- export — what the operator keeps beside the key --------------------------------- + say("export — every root secret, sealed to the operator key, written beside it") + exported, err := ExportRootSecrets(ctx, o, permanentControl, say) + result.RootExport = exported + if err != nil { + return result, failed(StepExtras, err) + } + say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " + "sits on its private network, and filters what modules declared.") say("what remains is somebody else's: adding nodes, and assigning what they should run.") diff --git a/internal/bootstrap/operator.go b/internal/bootstrap/operator.go new file mode 100644 index 0000000..0bc6776 --- /dev/null +++ b/internal/bootstrap/operator.go @@ -0,0 +1,107 @@ +package bootstrap + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/identity" +) + +// The operator's sealing key: made at genesis, before the mesh is told any secret. +// +// Every secret a module holds for itself is sealed to the node that uses it; from here on it is +// sealed to this key as well, and the vault keeps those copies (novox/hq ADR 0085, amended). The +// private half is written once, beside the produced bundle, and given to nothing: the mesh +// records the public half and can open nothing it seals to it. The operator copies the file off +// the machine and keeps it — it is what recovers the mesh's root secrets when a node cannot. +// +// **Before enrolment's first `secret accept`**, or the credentials genesis made would be sealed +// to the node alone and be exactly as unrecoverable as the constants they replaced. + +// OperatorKeyFile is where the private half is written, beside the bundle. +func OperatorKeyFile(o Options) string { + return filepath.Join(filepath.Dir(o.Out), "operator.key") +} + +// RootExportFile is where the export of every operator-sealed secret is written at the end. +func RootExportFile(o Options) string { + return filepath.Join(filepath.Dir(o.Out), "root-secrets.export.json") +} + +type OperatorKey struct { + Path string + Fingerprint string + Made bool +} + +// MakeOperatorKey makes the key if this machine has none, and tells the mesh its public half. +func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say func(string)) (OperatorKey, error) { + out := OperatorKey{Path: OperatorKeyFile(o)} + key, err := identity.LoadSealingKey(out.Path) + switch { + case err == nil: + say(" operator key already at " + out.Path + " — kept") + case os.IsNotExist(underlying(err)) || strings.Contains(err.Error(), "no sealing key at"): + key, err = identity.GenerateSealingKey() + if err != nil { + return out, err + } + if err := os.MkdirAll(filepath.Dir(out.Path), 0o755); err != nil { + return out, err + } + if err := os.WriteFile(out.Path, []byte(key.Private+"\n"), 0o600); err != nil { + return out, err + } + out.Made = true + default: + return out, err + } + sum := sha256.Sum256([]byte(key.Public)) + out.Fingerprint = "sha256:" + hex.EncodeToString(sum[:8]) + + if _, err := control.tell(ctx, "operator", "key", "set", key.Public); err != nil { + return out, err + } + if out.Made { + say(" operator key " + out.Fingerprint + " — private half at " + out.Path + " (0600)") + say(" COPY IT OFF THIS MACHINE AND KEEP IT: it opens the mesh's root secrets, and") + say(" nothing else does. The mesh holds only the public half.") + } else { + say(" operator key " + out.Fingerprint + " — the mesh seals its root secrets to it") + } + return out, nil +} + +func underlying(err error) error { + for { + next, ok := err.(interface{ Unwrap() error }) + if !ok || next.Unwrap() == nil { + return err + } + err = next.Unwrap() + } +} + +// ExportRootSecrets writes the export beside the operator key: every secret sealed to it, as +// ciphertext, and the honest list of what is not. What the vault keeps on its disk, kept once +// more by the person who holds the key. +func ExportRootSecrets(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + path := RootExportFile(o) + body, err := control.tell(ctx, "secret", "export") + if err != nil { + return path, err + } + if !strings.Contains(body, `"kept"`) { + return path, fmt.Errorf("`secret export` did not produce an export:\n%s", body) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + return path, err + } + say(" exported " + path + " (0600) — ciphertext, sealed to the operator key; keep it with the key") + return path, nil +} diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index a74ce76..5ef8734 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "os" "strings" "github.com/novox/mesh-host/internal/declaration" @@ -116,6 +117,123 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, return nil } +func readCredentialFile(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimRight(string(raw), "\r\n") + if value == "" { + return "", fmt.Errorf("%s is empty", path) + } + return value, nil +} + +// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same +// shape as InstallStore, for the same reasons. The administrator's password is the one genesis +// gave the image's default account (rootsecrets.go), carried in through `secret accept` so the +// module's provisioner can reach the management API as it. +func InstallBroker(ctx context.Context, o Options, control controlPlane, + foundation *declaration.Declaration, say func(string)) error { + + const module = "lavinmq" + manifest, err := readManifest(o.Catalogue, module) + if err != nil { + return err + } + broker, err := brokerIn(foundation) + if err != nil { + return err + } + if err := serverMatchesFoundation(manifest, broker, module); err != nil { + return err + } + say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged") + + remote := "/" + module + "-module.json" + if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { + return err + } + if _, err := control.tell(ctx, "module", "add", remote); err != nil { + return err + } + say(" registered " + module) + if o.CatalogSource.Repository == "" { + return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from", module) + } + say(" building " + module + " (the provisioner; the server is adopted, not built)") + if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, + "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { + return err + } + if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { + say(" no account " + module + " — it declares nothing to say on the broker") + } else { + say(" account issued " + module) + } + if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { + return err + } + + value, err := readCredentialFile(BrokerAdminFile) + if err != nil { + return fmt.Errorf("the broker's administrator password genesis made is not at %s: %w", BrokerAdminFile, err) + } + at := "/accepting-admin" + if err := control.carrying(ctx, "mesh-accepting-admin", []byte(value), at); err != nil { + return err + } + if _, err := control.tell(ctx, "secret", "accept", o.Node, module, "admin", "--from", at); err != nil { + return err + } + say(" accepted admin — the broker's administrator, as genesis made it") + + if _, err := pushNode(ctx, o, control, say); err != nil { + return err + } + say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module") + return nil +} + +// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to +// adopt: it is its own runtime, built from the catalogue like any provider, and from its first push +// it keeps the export of every operator-sealed secret on its own disk. +func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error { + const module = "mesh-vault" + manifest, err := readManifest(o.Catalogue, module) + if err != nil { + return err + } + remote := "/" + module + "-module.json" + if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { + return err + } + if _, err := control.tell(ctx, "module", "add", remote); err != nil { + return err + } + say(" registered " + module) + if o.CatalogSource.Repository == "" { + return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from", module) + } + say(" building " + module) + if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, + "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { + return err + } + if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { + return err + } + say(" account issued " + module) + if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { + return err + } + if _, err := pushNode(ctx, o, control, say); err != nil { + return err + } + say(" installed " + module + " — the operator-sealed export now lives on this machine, outside the store") + return nil +} + // storeIn finds the store container in the bundle this installer produced. func storeIn(d *declaration.Declaration) (*declaration.Container, error) { return foundationContainer(d, StoreID, "store") @@ -186,12 +304,17 @@ func deliverSuperuser(ctx context.Context, o Options, control controlPlane, modu store *declaration.Container, say func(string)) error { const secret = "superuser" - value := strings.TrimSpace(store.Env["POSTGRES_PASSWORD"]) + // Genesis made it and kept it in the file the store was raised from (rootsecrets.go); the + // template's environment variable is accepted too, for a bundle produced before that. + value, err := readCredentialFile(StoreSuperuserFile) + if err != nil { + value = strings.TrimSpace(store.Env["POSTGRES_PASSWORD"]) + } if value == "" { return fmt.Errorf( - "the foundation's store names no POSTGRES_PASSWORD, so the %s module has no superuser "+ - "to open it with — and the mesh cannot invent the one that already made the databases", - module) + "neither %s nor the foundation's store names the superuser password, so the %s module "+ + "has nothing to open the store with — and the mesh cannot invent the one that already "+ + "made the databases", StoreSuperuserFile, module) } at := "/accepting-" + secret if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go index 45eb470..cbfc8cd 100644 --- a/internal/bootstrap/rewrite.go +++ b/internal/bootstrap/rewrite.go @@ -316,17 +316,17 @@ func sortStrings(values []string) { // writeBundleFile puts the produced bundle where a person can read it, creating the directory it // lives in. // -// 0644, and that is deliberate: this file names an image and describes a foundation, and it holds -// the bootstrap credentials the template happens to carry — which are the same ones anybody can -// read in the template itself. It is meant to be read. What must not be world-readable is the -// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`). +// 0600: the produced bundle carries the credentials genesis made — the store's and the broker's, +// inside the temporary control plane's connection strings (novox/hq issue 071). It used to be +// 0644 and say so was fine because the template's credentials were the same ones anybody could +// read in the template; they are not any more. Still meant to be read, by root. func writeBundleFile(path string, content []byte) error { if dir := filepath.Dir(path); dir != "" && dir != "." { if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err) } } - if err := os.WriteFile(path, content, 0o644); err != nil { + if err := os.WriteFile(path, content, 0o600); err != nil { return fmt.Errorf( "cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+ "applying something nobody can read afterwards is how a machine becomes a mystery", diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go new file mode 100644 index 0000000..9cd6263 --- /dev/null +++ b/internal/bootstrap/rootsecrets.go @@ -0,0 +1,197 @@ +package bootstrap + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// The mesh's root credentials, made at genesis rather than copied from the template. +// +// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq +// issue 071). The store's superuser and the broker's administrator are the two credentials every +// other one rests on, and they were the two that were not secret: constants in a file anybody can +// read, carried into the mesh by `secret accept` and marked as something the mesh must never +// replace — which is correct for a credential that already created the databases, and made the +// well-known value permanent. +// +// So the installer makes them. Two random values, **made once and kept on this machine** at the +// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three +// adopts the store and the broker, `secret accept` carries in exactly the value the servers were +// raised with, and the host's later write of the sealed secret lands the same bytes in the same +// file. A second run finds the files and changes nothing, which is what lets the installer say +// "already done" about a store it must not restart. +// +// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a +// container's environment is in `docker inspect` for ever; the module that adopts the store +// declares the same file mount, so the two specs are one and the applier reconciles rather than +// recreates (phase3.go). The broker has no such file: its image's default administrator is changed +// in place by an action once the broker answers, and the produced bundle carries that action. +const ( + // StoreSuperuserFile is where the store's superuser password lives on the machine — the + // postgres module's own-secret path, so genesis and adoption write the same file. + StoreSuperuserFile = "/var/lib/postgres/superuser.secret" + // BrokerAdminFile is the same for the broker's administrator — the lavinmq module's. + BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret" + // BrokerAdminUser is the broker's administrator. The image's default account, kept by name + // and given a password that is not the image's default; a renamed account would have to be + // created before anything can authenticate, and the thing that creates accounts is the thing + // that has to authenticate first. + BrokerAdminUser = "guest" + + storeSuperuserMount = "/run/secrets/superuser" + + // What the template says, matched exactly. A template that says something else is a template + // this installer does not know how to make safe, and it says so rather than guessing. + templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` + templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]` + templateStoreURL = "postgres:bootstrap@" + templateBrokerURL = "guest:guest@" + templateBrokerReady = "\"verify\": [\"lavinmqctl\", \"status\"]\n }," + brokerAdminMarker = "/var/lib/lavinmq/.mesh-admin" +) + +// RootCredentials are the two values, and whether this run made them. +type RootCredentials struct { + Store, Broker string + StoreMade, BrokerMade bool +} + +// RootSecrets reads the credentials this machine already holds, or makes them. +// +// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the +// real one, and a machine that was only asked is not left holding half a genesis. +func RootSecrets(dryRun bool) (RootCredentials, error) { + var out RootCredentials + var err error + if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil { + return out, err + } + if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil { + return out, err + } + return out, nil +} + +func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { + raw, err := os.ReadFile(path) + if err == nil { + value = strings.TrimRight(string(raw), "\r\n") + if value == "" { + return "", false, fmt.Errorf("%s exists and is empty; move it aside to have one made", path) + } + return value, false, nil + } + if !os.IsNotExist(err) { + return "", false, err + } + value, err = freshSecret() + if err != nil { + return "", false, err + } + if dryRun { + return value, true, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", false, err + } + // Written whole and renamed into place, at 0600, owned by whoever runs the installer — root, + // which is also who the host runs as when it later writes the sealed copy here. + tmp := path + ".genesis" + if err := os.WriteFile(tmp, []byte(value+"\n"), 0o600); err != nil { + return "", false, err + } + if err := os.Rename(tmp, path); err != nil { + return "", false, err + } + return value, true, nil +} + +// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40 +// characters, URL-safe — it lands inside connection strings. +func freshSecret() (string, error) { + b := make([]byte, 30) + if _, err := rand.Read(b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +// RootRewrite says what RewriteRoot did to the bundle. +type RootRewrite struct { + StoreURLs, BrokerURLs int +} + +// RewriteRoot puts the made credentials into the produced bundle, in place of the template's. +// +// Byte for byte, like the image rewrite, so the file keeps its comments and a person can read what +// was applied. Every replacement is counted and a count of zero is refused: a template that no +// longer says what this expects is one whose credentials this would silently leave at the +// well-known values, which is the fault this exists to remove. +func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { + var out RootRewrite + bundle := r.Bundle + + // The store: a file, not an environment variable. + var err error + if bundle, err = replaceOnce(bundle, templateStorePassword, + `"POSTGRES_PASSWORD_FILE": "`+storeSuperuserMount+`"`, "the store's password"); err != nil { + return out, err + } + if bundle, err = replaceOnce(bundle, templateStoreVolumes, + `"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, + "the store's volumes"); err != nil { + return out, err + } + // Everything that dials the store or the broker with the template's credentials. + out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL)) + if out.StoreURLs == 0 { + return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL) + } + bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@")) + out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL)) + if out.BrokerURLs == 0 { + return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL) + } + bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@")) + + // The broker's administrator, changed once the broker answers and before anything dials it. + // Verified by a marker on the broker's own data volume, because the image carries nothing that + // can try a password from inside; what proves the password is the control plane answering + // over it, a few resources later. + action := templateBrokerReady + "\n" + + " {\n" + + " \"id\": \"broker-admin\",\n" + + " \"type\": \"action\",\n" + + " \"in\": \"mesh-broker\",\n" + + " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && touch " + brokerAdminMarker + "\"],\n" + + " \"verify\": [\"sh\", \"-c\", \"test -f " + brokerAdminMarker + "\"]\n" + + " }," + if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil { + return out, err + } + + parsed, err := declaration.ParseFileTrusted(bundle) + if err != nil { + return out, fmt.Errorf("the bundle stopped being a declaration after its credentials were rewritten, which is this installer's fault: %w", err) + } + r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources) + return out, nil +} + +func replaceOnce(in []byte, from, to, what string) ([]byte, error) { + switch n := bytes.Count(in, []byte(from)); n { + case 1: + return bytes.Replace(in, []byte(from), []byte(to), 1), nil + case 0: + return nil, fmt.Errorf("the template does not say %s the way this installer expects (%s), so it cannot be made safe here", what, from) + default: + return nil, fmt.Errorf("the template says %s %d times, and this installer expected once", what, n) + } +} diff --git a/internal/bootstrap/rootsecrets_test.go b/internal/bootstrap/rootsecrets_test.go new file mode 100644 index 0000000..ced776a --- /dev/null +++ b/internal/bootstrap/rootsecrets_test.go @@ -0,0 +1,123 @@ +package bootstrap + +import ( + "os" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// The produced bundle carries no well-known credential: the store reads its password from the +// file genesis made, every connection string names the made values, and the broker's default +// administrator is changed by an action before anything dials it (novox/hq issue 071). +func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"} + got, err := RewriteRoot(&r, creds) + if err != nil { + t.Fatal(err) + } + text := string(r.Bundle) + for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} { + if strings.Contains(text, gone) { + t.Errorf("the produced bundle still says %s", gone) + } + } + if got.StoreURLs < 3 || got.BrokerURLs < 2 { + t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs) + } + var store, action bool + for _, res := range r.Declaration.Resources { + switch x := res.(type) { + case *declaration.Container: + if x.Name != "mesh-store" { + continue + } + store = true + if _, has := x.Env["POSTGRES_PASSWORD"]; has { + t.Error("the store still takes its password from its environment") + } + if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount { + t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"]) + } + if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) { + t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes) + } + case *declaration.Action: + if x.ID != "broker-admin" { + continue + } + action = true + if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") { + t.Errorf("the broker-admin action is %v in %q", x.Command, x.In) + } + } + } + if !store || !action { + t.Fatalf("store=%v action=%v", store, action) + } + // The order matters: the broker's password changes after it answers and before the control + // plane, which dials it with the new one, is raised. + var readyAt, adminAt, controlAt int + for i, res := range r.Declaration.Resources { + switch res.Identity() { + case "broker-ready": + readyAt = i + case "broker-admin": + adminAt = i + case "control-plane": + controlAt = i + } + } + if !(readyAt < adminAt && adminAt < controlAt) { + t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt) + } +} + +// A template that no longer says what this expects is refused, not half-rewritten. +func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) { + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1) + r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil { + t.Fatal("a template with an unknown store password was rewritten") + } +} + +// Made once and kept: a second run reads the same value; a dry run writes nothing. +func TestRootSecretsAreKeptAcrossRuns(t *testing.T) { + dir := t.TempDir() + path := dir + "/superuser.secret" + first, made, err := keptOrMade(path, false) + if err != nil || !made || len(first) != 40 { + t.Fatalf("first: %q made=%v err=%v", first, made, err) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Errorf("mode %v", info.Mode().Perm()) + } + second, made, err := keptOrMade(path, false) + if err != nil || made || second != first { + t.Fatalf("second: %q made=%v err=%v", second, made, err) + } + dry := dir + "/dry.secret" + if _, made, err := keptOrMade(dry, true); err != nil || !made { + t.Fatal(err) + } + if _, err := os.Stat(dry); err == nil { + t.Fatal("a dry run wrote a secret") + } +} From 036af3cfdce0f3e3b558d075c52df39e50ec0663 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 01:11:52 +0200 Subject: [PATCH 2/4] The export is its own installer step, and the result names the operator files --- internal/bootstrap/bootstrap.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 278f0ae..a274327 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -64,6 +64,7 @@ const ( StepNetwork Step = "network" StepFilter Step = "filter" StepExtras Step = "extras" + StepExport Step = "export" ) // Steps in the order they happen, so a failure can say "step 2 of 11". @@ -84,7 +85,7 @@ var Steps = []Step{ // Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to // build, to say what it holds, on its network, filtering. They used to be things somebody // typed afterwards, which is how they went missing without anything complaining. - StepBase, StepStore, StepBroker, StepVault, StepCatalogue, StepNetwork, StepFilter, StepExtras, + StepBase, StepStore, StepBroker, StepVault, StepCatalogue, StepNetwork, StepFilter, StepExtras, StepExport, } // Error is a failure, named by the step it happened in. @@ -206,9 +207,9 @@ type Deps struct { type Result struct { // OperatorKey is where the operator's private key was written; OperatorKeyMade whether this // run made it. RootExport is where the operator-sealed export landed. - OperatorKey string - OperatorKeyMade bool - RootExport string + OperatorKey string `json:"operator-key,omitempty"` + OperatorKeyMade bool `json:"operator-key-made,omitempty"` + RootExport string `json:"root-export,omitempty"` System string `json:"system"` DryRun bool `json:"dry-run,omitempty"` @@ -686,7 +687,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro exported, err := ExportRootSecrets(ctx, o, permanentControl, say) result.RootExport = exported if err != nil { - return result, failed(StepExtras, err) + return result, failed(StepExport, err) } say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " + From 70d0f36896152a9ec03064978c87564e0dcda4fc Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 01:26:35 +0200 Subject: [PATCH 3/4] Installer review: secrets are staged privately, and a bundle is 0600 whether or not it existed From review: the store and broker passwords genesis makes were carried into the controller through a world-readable file in /tmp, a bundle left at 0644 by an earlier installer kept that mode while now holding them, a mesh raised by the old installer would have been handed new passwords its servers do not have, and the broker-admin action's marker did not depend on the value. Secrets now stage in a 0700 directory owned by the controller's account; the bundle is chmod'd; an existing store or broker volume with no credential file is refused by name; the marker holds the password's fingerprint. Also: one install path for the store, broker and vault, no error-string matching for the operator key, and no unreachable fallback for the superuser. --- internal/bootstrap/bootstrap.go | 7 +- internal/bootstrap/control.go | 2 +- internal/bootstrap/operator.go | 23 ++-- internal/bootstrap/phase3.go | 175 +++++++++------------------ internal/bootstrap/phase_packages.go | 2 +- internal/bootstrap/rewrite.go | 5 + internal/bootstrap/rootsecrets.go | 76 ++++++++++-- internal/bootstrap/talk.go | 40 +++++- 8 files changed, 175 insertions(+), 155 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a274327..7f06093 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -394,6 +394,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if err != nil { return result, failed(StepBundle, err) } + if err := RefuseExistingServers(ctx, d.Run, creds); err != nil { + return result, failed(StepBundle, err) + } root, err := RewriteRoot(&rewritten, creds) if err != nil { return result, failed(StepBundle, err) @@ -644,13 +647,13 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepStore, err) } - // ---- 14b. broker ---------------------------------------------------------------------- + // ---- broker --------------------------------------------------------------------------- say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two") if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { return result, failed(StepBroker, err) } - // ---- 14c. vault ----------------------------------------------------------------------- + // ---- vault ---------------------------------------------------------------------------- // A foundation module (novox/hq ADR 0085, amended): it keeps every operator-sealed secret on // its own disk, outside the store, from the first push that carries one. say("vault — the mesh's secrets get an owner, and their operator-sealed copies a place to live") diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go index 3125dd0..e806a44 100644 --- a/internal/bootstrap/control.go +++ b/internal/bootstrap/control.go @@ -274,7 +274,7 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. at := "/accepting-" + secret - if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { + if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, diff --git a/internal/bootstrap/operator.go b/internal/bootstrap/operator.go index 0bc6776..761689c 100644 --- a/internal/bootstrap/operator.go +++ b/internal/bootstrap/operator.go @@ -42,11 +42,14 @@ type OperatorKey struct { // MakeOperatorKey makes the key if this machine has none, and tells the mesh its public half. func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say func(string)) (OperatorKey, error) { out := OperatorKey{Path: OperatorKeyFile(o)} - key, err := identity.LoadSealingKey(out.Path) - switch { - case err == nil: + var key identity.SealingKey + if _, err := os.Stat(out.Path); err == nil { + key, err = identity.LoadSealingKey(out.Path) + if err != nil { + return out, err + } say(" operator key already at " + out.Path + " — kept") - case os.IsNotExist(underlying(err)) || strings.Contains(err.Error(), "no sealing key at"): + } else if os.IsNotExist(err) { key, err = identity.GenerateSealingKey() if err != nil { return out, err @@ -58,7 +61,7 @@ func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say f return out, err } out.Made = true - default: + } else { return out, err } sum := sha256.Sum256([]byte(key.Public)) @@ -77,16 +80,6 @@ func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say f return out, nil } -func underlying(err error) error { - for { - next, ok := err.(interface{ Unwrap() error }) - if !ok || next.Unwrap() == nil { - return err - } - err = next.Unwrap() - } -} - // ExportRootSecrets writes the export beside the operator key: every secret sealed to it, as // ciphertext, and the honest list of what is not. What the vault keeps on its disk, kept once // more by the person who holds the key. diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index 5ef8734..e39b9e3 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strings" "github.com/novox/mesh-host/internal/declaration" @@ -73,6 +72,27 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, } say(" adopting " + store.Name + " — the store the foundation raised, unchanged") + // The superuser is the foundation's, made at genesis — carried in before the push, or the push + // would seal random bytes where a working password has to be and the provisioner would not open + // the store it is meant to manage. + if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", + func() error { + return deliverCredential(ctx, o, control, module, "superuser", StoreSuperuserFile, "the store's superuser", say) + }, + say); err != nil { + return err + } + say(" adopted mesh-store — the foundation's store is now the " + module + " module") + return nil +} + +// installProvider registers, builds, issues, assigns and pushes one catalogue module, with one +// thing done just before the push — the moment a credential the mesh could not have made has to +// be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker +// and the vault each need it or need the shape, and three copies of it drifted. +func installProvider(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, buildNote string, beforePush func() error, say func(string)) error { + remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { return err @@ -87,7 +107,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, "the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where a builder "+ "clones it", module) } - say(" building " + module + " (the provisioner; the server is adopted, not built)") + say(strings.TrimRight(" building "+module+" "+buildNote, " ")) if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { return err @@ -102,37 +122,40 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } - - // The superuser is the foundation's, made at genesis — carried in before the push, or the push - // would seal random bytes where a working password has to be and the provisioner would not open - // the store it is meant to manage. - if err := deliverSuperuser(ctx, o, control, module, store, say); err != nil { - return err + if beforePush != nil { + if err := beforePush(); err != nil { + return err + } } - - if _, err := pushNode(ctx, o, control, say); err != nil { - return err - } - say(" adopted mesh-store — the foundation's store is now the " + module + " module") - return nil + _, err := pushNode(ctx, o, control, say) + return err } -func readCredentialFile(path string) (string, error) { - raw, err := os.ReadFile(path) +// deliverCredential carries a credential genesis made into a module as its own secret, through +// `secret accept`: the mesh cannot invent the value a running server already has. +func deliverCredential(ctx context.Context, o Options, control controlPlane, module, secret, file, what string, + say func(string)) error { + + value, err := readCredentialFile(file) if err != nil { - return "", err + return fmt.Errorf("%s is not at %s, so the %s module has nothing to open its server with — "+ + "and the mesh cannot invent the one the server already has: %w", what, file, module, err) } - value := strings.TrimRight(string(raw), "\r\n") - if value == "" { - return "", fmt.Errorf("%s is empty", path) + at := "/accepting-" + secret + if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { + return err } - return value, nil + if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { + return err + } + say(" accepted " + secret + " — " + what + ", as genesis made it") + return nil } // InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same // shape as InstallStore, for the same reasons. The administrator's password is the one genesis -// gave the image's default account (rootsecrets.go), carried in through `secret accept` so the -// module's provisioner can reach the management API as it. +// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can +// reach the management API as it. func InstallBroker(ctx context.Context, o Options, control controlPlane, foundation *declaration.Declaration, say func(string)) error { @@ -149,46 +172,11 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, return err } say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged") - - remote := "/" + module + "-module.json" - if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "add", remote); err != nil { - return err - } - say(" registered " + module) - if o.CatalogSource.Repository == "" { - return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from", module) - } - say(" building " + module + " (the provisioner; the server is adopted, not built)") - if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, - "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { - say(" no account " + module + " — it declares nothing to say on the broker") - } else { - say(" account issued " + module) - } - if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { - return err - } - - value, err := readCredentialFile(BrokerAdminFile) - if err != nil { - return fmt.Errorf("the broker's administrator password genesis made is not at %s: %w", BrokerAdminFile, err) - } - at := "/accepting-admin" - if err := control.carrying(ctx, "mesh-accepting-admin", []byte(value), at); err != nil { - return err - } - if _, err := control.tell(ctx, "secret", "accept", o.Node, module, "admin", "--from", at); err != nil { - return err - } - say(" accepted admin — the broker's administrator, as genesis made it") - - if _, err := pushNode(ctx, o, control, say); err != nil { + if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", + func() error { + return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say) + }, + say); err != nil { return err } say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module") @@ -196,38 +184,15 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, } // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to -// adopt: it is its own runtime, built from the catalogue like any provider, and from its first push -// it keeps the export of every operator-sealed secret on its own disk. +// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider, +// and from its first push it keeps the export of every operator-sealed secret on its own disk. func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error { const module = "mesh-vault" manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } - remote := "/" + module + "-module.json" - if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "add", remote); err != nil { - return err - } - say(" registered " + module) - if o.CatalogSource.Repository == "" { - return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from", module) - } - say(" building " + module) - if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, - "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { - return err - } - say(" account issued " + module) - if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { - return err - } - if _, err := pushNode(ctx, o, control, say); err != nil { + if err := installProvider(ctx, o, control, module, manifest, "", nil, say); err != nil { return err } say(" installed " + module + " — the operator-sealed export now lives on this machine, outside the store") @@ -294,35 +259,3 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu "store with. Its server container has to carry the name the foundation raised", module, store.Name) } - -// deliverSuperuser carries the store's superuser password into the module. -// -// It is the foundation's, set on the bundle's store container at genesis; the mesh cannot invent a -// credential that already made the databases, so it goes in through `secret accept`, exactly as the -// control plane's store connections do (control.go deliverStores). -func deliverSuperuser(ctx context.Context, o Options, control controlPlane, module string, - store *declaration.Container, say func(string)) error { - - const secret = "superuser" - // Genesis made it and kept it in the file the store was raised from (rootsecrets.go); the - // template's environment variable is accepted too, for a bundle produced before that. - value, err := readCredentialFile(StoreSuperuserFile) - if err != nil { - value = strings.TrimSpace(store.Env["POSTGRES_PASSWORD"]) - } - if value == "" { - return fmt.Errorf( - "neither %s nor the foundation's store names the superuser password, so the %s module "+ - "has nothing to open the store with — and the mesh cannot invent the one that already "+ - "made the databases", StoreSuperuserFile, module) - } - at := "/accepting-" + secret - if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { - return err - } - if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { - return err - } - say(" accepted " + secret + " — the store's superuser, as the foundation made it") - return nil -} diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index d7f24ca..1a4ceed 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -270,7 +270,7 @@ func packagePasswords() (db, admin, builder string, err error) { func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string, say func(string)) error { at := "/accepting-npm-password" - if err := control.carrying(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { + if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { return err } if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil { diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go index cbfc8cd..456b219 100644 --- a/internal/bootstrap/rewrite.go +++ b/internal/bootstrap/rewrite.go @@ -332,5 +332,10 @@ func writeBundleFile(path string, content []byte) error { "applying something nobody can read afterwards is how a machine becomes a mystery", path, err) } + // The mode above applies only when the file is created. A bundle an earlier installer left at + // 0644 would keep that while now carrying real credentials, with this function saying 0600. + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("cannot make the produced bundle %s readable by root alone: %w", path, err) + } return nil } diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go index 9cd6263..245cf20 100644 --- a/internal/bootstrap/rootsecrets.go +++ b/internal/bootstrap/rootsecrets.go @@ -2,8 +2,11 @@ package bootstrap import ( "bytes" + "context" "crypto/rand" + "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" "os" "path/filepath" @@ -80,12 +83,8 @@ func RootSecrets(dryRun bool) (RootCredentials, error) { } func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { - raw, err := os.ReadFile(path) + value, err = readCredentialFile(path) if err == nil { - value = strings.TrimRight(string(raw), "\r\n") - if value == "" { - return "", false, fmt.Errorf("%s exists and is empty; move it aside to have one made", path) - } return value, false, nil } if !os.IsNotExist(err) { @@ -113,6 +112,27 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { return value, true, nil } +// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone. +// Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable". +func readCredentialFile(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimRight(string(raw), "\r\n") + if value == "" { + return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path) + } + return value, nil +} + +// credentialFingerprint names a credential without being one — what the broker-admin action +// leaves on the broker's volume, so its verify holds for this value and not for any value. +func credentialFingerprint(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:8]) +} + // freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40 // characters, URL-safe — it lands inside connection strings. func freshSecret() (string, error) { @@ -123,6 +143,40 @@ func freshSecret() (string, error) { return base64.RawURLEncoding.EncodeToString(b), nil } +// RefuseExistingServers stops a run that would put a made credential in front of a server raised +// by an earlier installer with the template's. +// +// The store's password is set by initdb, once, on an empty volume; the broker's by the action +// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no +// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a +// bundle that dials with passwords the servers do not have — failing three steps later, in the +// schemas, with nothing pointing back here. Refused by name instead, with the way forward. +func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error { + for _, check := range []struct { + made bool + volume string + what string + file string + }{ + {c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile}, + {c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile}, + } { + if !check.made { + continue + } + if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil { + continue // no such volume: a fresh machine, which is the case this installer makes + } + return fmt.Errorf( + "this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+ + "by an earlier installer with the template's password. A new one made here would not open it. "+ + "Put the password the %s has into %s (0600, the value alone) and run again; then change it "+ + "on the server and accept the new value — this installer does not rotate a running %s", + check.what, check.volume, check.file, check.what, check.file, check.what) + } + return nil +} + // RootRewrite says what RewriteRoot did to the bundle. type RootRewrite struct { StoreURLs, BrokerURLs int @@ -162,16 +216,18 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@")) // The broker's administrator, changed once the broker answers and before anything dials it. - // Verified by a marker on the broker's own data volume, because the image carries nothing that - // can try a password from inside; what proves the password is the control plane answering - // over it, a few resources later. + // Verified by a marker on the broker's own data volume holding this password's fingerprint — + // the image carries nothing that can try a password from inside, and a marker that merely + // existed would let a regenerated password go unapplied for ever. What proves the password + // works is the control plane answering over it, a few resources later. + fp := credentialFingerprint(c.Broker) action := templateBrokerReady + "\n" + " {\n" + " \"id\": \"broker-admin\",\n" + " \"type\": \"action\",\n" + " \"in\": \"mesh-broker\",\n" + - " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && touch " + brokerAdminMarker + "\"],\n" + - " \"verify\": [\"sh\", \"-c\", \"test -f " + brokerAdminMarker + "\"]\n" + + " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && printf %s " + fp + " > " + brokerAdminMarker + "\"],\n" + + " \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" + " }," if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil { return out, err diff --git a/internal/bootstrap/talk.go b/internal/bootstrap/talk.go index 3b5b029..a9c4a8f 100644 --- a/internal/bootstrap/talk.go +++ b/internal/bootstrap/talk.go @@ -82,11 +82,9 @@ func (c controlPlane) carry(ctx context.Context, local, remote string) error { // landed unreadable, `secret accept` failed with `permission denied`, and what depended on it // crash-looped on material it never received. There is no shell in the image to chown it with. // -// What goes through here is a module manifest and a store connection string. The connection is the -// same value the produced bundle already holds in the clear — a foundation names its own bootstrap -// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The -// file on the machine is removed at once, and the copy inside the container goes when the -// container does, which for the temporary control plane is step 10. +// What goes through here is a module manifest — public, the same bytes as in the catalogue. A +// value that is secret goes through carryingSecret below. The file on the machine is removed at +// once, and the copy inside the container goes when the container does. func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error { local := filepath.Join(os.TempDir(), name) if err := os.WriteFile(local, content, 0o644); err != nil { @@ -96,6 +94,38 @@ func (c controlPlane) carrying(ctx context.Context, name string, content []byte, return c.carry(ctx, local, remote) } +// controlPlaneUID is the account the control plane's image runs as — `USER 65534:65534` in its +// Dockerfile — and so the only account inside the container that needs to read what is carried in. +const controlPlaneUID = 65534 + +// carryingSecret is carrying for a value that is a secret: staged in a directory only root can +// enter, at 0600, owned by the control plane's own account — which `docker cp` keeps, so inside +// the container the file is readable by the process that must read it and by nobody else. Since +// genesis makes the mesh's root credentials rather than copying the template's (rootsecrets.go), +// a store connection string or a broker password carried this way is a real secret, and 0644 in a +// shared temporary directory would hand it to any local user for the length of the copy. +func (c controlPlane) carryingSecret(ctx context.Context, name string, content []byte, remote string) error { + dir, err := os.MkdirTemp("", "mesh-carrying-") + if err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + defer os.RemoveAll(dir) + local := filepath.Join(dir, name) + if err := os.WriteFile(local, content, 0o600); err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + if err := os.Chown(local, controlPlaneUID, controlPlaneUID); err != nil { + // Not root — a test, or an installer run as a user, which no real genesis is. The + // directory is 0700, so nobody else on the machine can reach the file either way; inside + // the container the only account is the control plane's, so 0644 there is read by it and + // by nothing else. The narrower ownership is taken whenever it can be. + if err := os.Chmod(local, 0o644); err != nil { + return err + } + } + return c.carry(ctx, local, remote) +} + func indent(s string) string { if s == "" { return "" From d756effc33aebc17089384433b54aa63dbeee4b3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 01:32:51 +0200 Subject: [PATCH 4/4] The broker-admin marker ends in a newline, and the transcript never says a credential MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verify reads the marker with the shell's read, which fails at end of file without a line ending; the action ran and its verify said no. And the applier reports each action with its command line, two of which now carry the real store and broker passwords — the installer masks the values it made in everything it says. --- internal/bootstrap/bootstrap.go | 2 ++ internal/bootstrap/rootsecrets.go | 20 ++++++++++-- internal/bootstrap/rootsecrets_test.go | 44 ++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 7f06093..049112a 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -397,6 +397,8 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if err := RefuseExistingServers(ctx, d.Run, creds); err != nil { return result, failed(StepBundle, err) } + // From here on nothing this installer says contains the values it just made. + say = Masking(say, creds) root, err := RewriteRoot(&rewritten, creds) if err != nil { return result, failed(StepBundle, err) diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go index 245cf20..06a81f2 100644 --- a/internal/bootstrap/rootsecrets.go +++ b/internal/bootstrap/rootsecrets.go @@ -143,6 +143,20 @@ func freshSecret() (string, error) { return base64.RawURLEncoding.EncodeToString(b), nil } +// Masking makes a reporter that never says the credentials this run made. +// +// The applier reports each action with its command line, and two of them now carry a real +// password — the context schemas' connection strings and the broker's change_password. Those +// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The +// exact values are known here, so they are replaced wherever they appear, in every line said. +func Masking(say func(string), c RootCredentials) func(string) { + replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…") + if c.Store == "" || c.Broker == "" { + return say + } + return func(line string) { say(replacer.Replace(line)) } +} + // RefuseExistingServers stops a run that would put a made credential in front of a server raised // by an earlier installer with the template's. // @@ -219,14 +233,16 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { // Verified by a marker on the broker's own data volume holding this password's fingerprint — // the image carries nothing that can try a password from inside, and a marker that merely // existed would let a regenerated password go unapplied for ever. What proves the password - // works is the control plane answering over it, a few resources later. + // works is the control plane answering over it, a few resources later. The marker ends in a + // newline because the verify reads it with the shell's `read`, which fails at end of file + // without one — an action that ran and a verify that said no, once, in the lab. fp := credentialFingerprint(c.Broker) action := templateBrokerReady + "\n" + " {\n" + " \"id\": \"broker-admin\",\n" + " \"type\": \"action\",\n" + " \"in\": \"mesh-broker\",\n" + - " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && printf %s " + fp + " > " + brokerAdminMarker + "\"],\n" + + " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" + " \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" + " }," if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil { diff --git a/internal/bootstrap/rootsecrets_test.go b/internal/bootstrap/rootsecrets_test.go index ced776a..c23f864 100644 --- a/internal/bootstrap/rootsecrets_test.go +++ b/internal/bootstrap/rootsecrets_test.go @@ -121,3 +121,47 @@ func TestRootSecretsAreKeptAcrossRuns(t *testing.T) { t.Fatal("a dry run wrote a secret") } } + +// Nothing the installer says after making the credentials contains them. +func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) { + var said []string + say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"}) + say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)") + say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)") + for _, l := range said { + if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") { + t.Errorf("said a credential: %s", l) + } + } + if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") { + t.Errorf("the lines were not the same lines with the values masked: %v", said) + } +} + +// The broker-admin marker is written with a line ending, because the verify reads it with `read`. +func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) { + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { + t.Fatal(err) + } + for _, res := range r.Declaration.Resources { + a, ok := res.(*declaration.Action) + if !ok || a.ID != "broker-admin" { + continue + } + cmd := strings.Join(a.Command, " ") + if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") { + t.Errorf("the marker is written without a line ending: %s", cmd) + } + if !strings.Contains(strings.Join(a.Verify, " "), "read m <") { + t.Errorf("the verify does not read the marker: %v", a.Verify) + } + } +}