From 70d0f36896152a9ec03064978c87564e0dcda4fc Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 01:26:35 +0200 Subject: [PATCH] Installer review: secrets are staged privately, and a bundle is 0600 whether or not it existed From review: the store and broker passwords genesis makes were carried into the controller through a world-readable file in /tmp, a bundle left at 0644 by an earlier installer kept that mode while now holding them, a mesh raised by the old installer would have been handed new passwords its servers do not have, and the broker-admin action's marker did not depend on the value. Secrets now stage in a 0700 directory owned by the controller's account; the bundle is chmod'd; an existing store or broker volume with no credential file is refused by name; the marker holds the password's fingerprint. Also: one install path for the store, broker and vault, no error-string matching for the operator key, and no unreachable fallback for the superuser. --- internal/bootstrap/bootstrap.go | 7 +- internal/bootstrap/control.go | 2 +- internal/bootstrap/operator.go | 23 ++-- internal/bootstrap/phase3.go | 175 +++++++++------------------ internal/bootstrap/phase_packages.go | 2 +- internal/bootstrap/rewrite.go | 5 + internal/bootstrap/rootsecrets.go | 76 ++++++++++-- internal/bootstrap/talk.go | 40 +++++- 8 files changed, 175 insertions(+), 155 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a274327..7f06093 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -394,6 +394,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if err != nil { return result, failed(StepBundle, err) } + if err := RefuseExistingServers(ctx, d.Run, creds); err != nil { + return result, failed(StepBundle, err) + } root, err := RewriteRoot(&rewritten, creds) if err != nil { return result, failed(StepBundle, err) @@ -644,13 +647,13 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepStore, err) } - // ---- 14b. broker ---------------------------------------------------------------------- + // ---- broker --------------------------------------------------------------------------- say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two") if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { return result, failed(StepBroker, err) } - // ---- 14c. vault ----------------------------------------------------------------------- + // ---- vault ---------------------------------------------------------------------------- // A foundation module (novox/hq ADR 0085, amended): it keeps every operator-sealed secret on // its own disk, outside the store, from the first push that carries one. say("vault — the mesh's secrets get an owner, and their operator-sealed copies a place to live") diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go index 3125dd0..e806a44 100644 --- a/internal/bootstrap/control.go +++ b/internal/bootstrap/control.go @@ -274,7 +274,7 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. at := "/accepting-" + secret - if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { + if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, diff --git a/internal/bootstrap/operator.go b/internal/bootstrap/operator.go index 0bc6776..761689c 100644 --- a/internal/bootstrap/operator.go +++ b/internal/bootstrap/operator.go @@ -42,11 +42,14 @@ type OperatorKey struct { // MakeOperatorKey makes the key if this machine has none, and tells the mesh its public half. func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say func(string)) (OperatorKey, error) { out := OperatorKey{Path: OperatorKeyFile(o)} - key, err := identity.LoadSealingKey(out.Path) - switch { - case err == nil: + var key identity.SealingKey + if _, err := os.Stat(out.Path); err == nil { + key, err = identity.LoadSealingKey(out.Path) + if err != nil { + return out, err + } say(" operator key already at " + out.Path + " — kept") - case os.IsNotExist(underlying(err)) || strings.Contains(err.Error(), "no sealing key at"): + } else if os.IsNotExist(err) { key, err = identity.GenerateSealingKey() if err != nil { return out, err @@ -58,7 +61,7 @@ func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say f return out, err } out.Made = true - default: + } else { return out, err } sum := sha256.Sum256([]byte(key.Public)) @@ -77,16 +80,6 @@ func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say f return out, nil } -func underlying(err error) error { - for { - next, ok := err.(interface{ Unwrap() error }) - if !ok || next.Unwrap() == nil { - return err - } - err = next.Unwrap() - } -} - // ExportRootSecrets writes the export beside the operator key: every secret sealed to it, as // ciphertext, and the honest list of what is not. What the vault keeps on its disk, kept once // more by the person who holds the key. diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index 5ef8734..e39b9e3 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strings" "github.com/novox/mesh-host/internal/declaration" @@ -73,6 +72,27 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, } say(" adopting " + store.Name + " — the store the foundation raised, unchanged") + // The superuser is the foundation's, made at genesis — carried in before the push, or the push + // would seal random bytes where a working password has to be and the provisioner would not open + // the store it is meant to manage. + if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", + func() error { + return deliverCredential(ctx, o, control, module, "superuser", StoreSuperuserFile, "the store's superuser", say) + }, + say); err != nil { + return err + } + say(" adopted mesh-store — the foundation's store is now the " + module + " module") + return nil +} + +// installProvider registers, builds, issues, assigns and pushes one catalogue module, with one +// thing done just before the push — the moment a credential the mesh could not have made has to +// be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker +// and the vault each need it or need the shape, and three copies of it drifted. +func installProvider(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, buildNote string, beforePush func() error, say func(string)) error { + remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { return err @@ -87,7 +107,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, "the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where a builder "+ "clones it", module) } - say(" building " + module + " (the provisioner; the server is adopted, not built)") + say(strings.TrimRight(" building "+module+" "+buildNote, " ")) if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { return err @@ -102,37 +122,40 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } - - // The superuser is the foundation's, made at genesis — carried in before the push, or the push - // would seal random bytes where a working password has to be and the provisioner would not open - // the store it is meant to manage. - if err := deliverSuperuser(ctx, o, control, module, store, say); err != nil { - return err + if beforePush != nil { + if err := beforePush(); err != nil { + return err + } } - - if _, err := pushNode(ctx, o, control, say); err != nil { - return err - } - say(" adopted mesh-store — the foundation's store is now the " + module + " module") - return nil + _, err := pushNode(ctx, o, control, say) + return err } -func readCredentialFile(path string) (string, error) { - raw, err := os.ReadFile(path) +// deliverCredential carries a credential genesis made into a module as its own secret, through +// `secret accept`: the mesh cannot invent the value a running server already has. +func deliverCredential(ctx context.Context, o Options, control controlPlane, module, secret, file, what string, + say func(string)) error { + + value, err := readCredentialFile(file) if err != nil { - return "", err + return fmt.Errorf("%s is not at %s, so the %s module has nothing to open its server with — "+ + "and the mesh cannot invent the one the server already has: %w", what, file, module, err) } - value := strings.TrimRight(string(raw), "\r\n") - if value == "" { - return "", fmt.Errorf("%s is empty", path) + at := "/accepting-" + secret + if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { + return err } - return value, nil + if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { + return err + } + say(" accepted " + secret + " — " + what + ", as genesis made it") + return nil } // InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same // shape as InstallStore, for the same reasons. The administrator's password is the one genesis -// gave the image's default account (rootsecrets.go), carried in through `secret accept` so the -// module's provisioner can reach the management API as it. +// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can +// reach the management API as it. func InstallBroker(ctx context.Context, o Options, control controlPlane, foundation *declaration.Declaration, say func(string)) error { @@ -149,46 +172,11 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, return err } say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged") - - remote := "/" + module + "-module.json" - if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "add", remote); err != nil { - return err - } - say(" registered " + module) - if o.CatalogSource.Repository == "" { - return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from", module) - } - say(" building " + module + " (the provisioner; the server is adopted, not built)") - if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, - "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { - say(" no account " + module + " — it declares nothing to say on the broker") - } else { - say(" account issued " + module) - } - if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { - return err - } - - value, err := readCredentialFile(BrokerAdminFile) - if err != nil { - return fmt.Errorf("the broker's administrator password genesis made is not at %s: %w", BrokerAdminFile, err) - } - at := "/accepting-admin" - if err := control.carrying(ctx, "mesh-accepting-admin", []byte(value), at); err != nil { - return err - } - if _, err := control.tell(ctx, "secret", "accept", o.Node, module, "admin", "--from", at); err != nil { - return err - } - say(" accepted admin — the broker's administrator, as genesis made it") - - if _, err := pushNode(ctx, o, control, say); err != nil { + if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", + func() error { + return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say) + }, + say); err != nil { return err } say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module") @@ -196,38 +184,15 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, } // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to -// adopt: it is its own runtime, built from the catalogue like any provider, and from its first push -// it keeps the export of every operator-sealed secret on its own disk. +// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider, +// and from its first push it keeps the export of every operator-sealed secret on its own disk. func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error { const module = "mesh-vault" manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } - remote := "/" + module + "-module.json" - if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "add", remote); err != nil { - return err - } - say(" registered " + module) - if o.CatalogSource.Repository == "" { - return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from", module) - } - say(" building " + module) - if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, - "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { - return err - } - if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { - return err - } - say(" account issued " + module) - if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { - return err - } - if _, err := pushNode(ctx, o, control, say); err != nil { + if err := installProvider(ctx, o, control, module, manifest, "", nil, say); err != nil { return err } say(" installed " + module + " — the operator-sealed export now lives on this machine, outside the store") @@ -294,35 +259,3 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu "store with. Its server container has to carry the name the foundation raised", module, store.Name) } - -// deliverSuperuser carries the store's superuser password into the module. -// -// It is the foundation's, set on the bundle's store container at genesis; the mesh cannot invent a -// credential that already made the databases, so it goes in through `secret accept`, exactly as the -// control plane's store connections do (control.go deliverStores). -func deliverSuperuser(ctx context.Context, o Options, control controlPlane, module string, - store *declaration.Container, say func(string)) error { - - const secret = "superuser" - // Genesis made it and kept it in the file the store was raised from (rootsecrets.go); the - // template's environment variable is accepted too, for a bundle produced before that. - value, err := readCredentialFile(StoreSuperuserFile) - if err != nil { - value = strings.TrimSpace(store.Env["POSTGRES_PASSWORD"]) - } - if value == "" { - return fmt.Errorf( - "neither %s nor the foundation's store names the superuser password, so the %s module "+ - "has nothing to open the store with — and the mesh cannot invent the one that already "+ - "made the databases", StoreSuperuserFile, module) - } - at := "/accepting-" + secret - if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { - return err - } - if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { - return err - } - say(" accepted " + secret + " — the store's superuser, as the foundation made it") - return nil -} diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index d7f24ca..1a4ceed 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -270,7 +270,7 @@ func packagePasswords() (db, admin, builder string, err error) { func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string, say func(string)) error { at := "/accepting-npm-password" - if err := control.carrying(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { + if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { return err } if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil { diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go index cbfc8cd..456b219 100644 --- a/internal/bootstrap/rewrite.go +++ b/internal/bootstrap/rewrite.go @@ -332,5 +332,10 @@ func writeBundleFile(path string, content []byte) error { "applying something nobody can read afterwards is how a machine becomes a mystery", path, err) } + // The mode above applies only when the file is created. A bundle an earlier installer left at + // 0644 would keep that while now carrying real credentials, with this function saying 0600. + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("cannot make the produced bundle %s readable by root alone: %w", path, err) + } return nil } diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go index 9cd6263..245cf20 100644 --- a/internal/bootstrap/rootsecrets.go +++ b/internal/bootstrap/rootsecrets.go @@ -2,8 +2,11 @@ package bootstrap import ( "bytes" + "context" "crypto/rand" + "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" "os" "path/filepath" @@ -80,12 +83,8 @@ func RootSecrets(dryRun bool) (RootCredentials, error) { } func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { - raw, err := os.ReadFile(path) + value, err = readCredentialFile(path) if err == nil { - value = strings.TrimRight(string(raw), "\r\n") - if value == "" { - return "", false, fmt.Errorf("%s exists and is empty; move it aside to have one made", path) - } return value, false, nil } if !os.IsNotExist(err) { @@ -113,6 +112,27 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { return value, true, nil } +// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone. +// Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable". +func readCredentialFile(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimRight(string(raw), "\r\n") + if value == "" { + return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path) + } + return value, nil +} + +// credentialFingerprint names a credential without being one — what the broker-admin action +// leaves on the broker's volume, so its verify holds for this value and not for any value. +func credentialFingerprint(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:8]) +} + // freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40 // characters, URL-safe — it lands inside connection strings. func freshSecret() (string, error) { @@ -123,6 +143,40 @@ func freshSecret() (string, error) { return base64.RawURLEncoding.EncodeToString(b), nil } +// RefuseExistingServers stops a run that would put a made credential in front of a server raised +// by an earlier installer with the template's. +// +// The store's password is set by initdb, once, on an empty volume; the broker's by the action +// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no +// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a +// bundle that dials with passwords the servers do not have — failing three steps later, in the +// schemas, with nothing pointing back here. Refused by name instead, with the way forward. +func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error { + for _, check := range []struct { + made bool + volume string + what string + file string + }{ + {c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile}, + {c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile}, + } { + if !check.made { + continue + } + if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil { + continue // no such volume: a fresh machine, which is the case this installer makes + } + return fmt.Errorf( + "this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+ + "by an earlier installer with the template's password. A new one made here would not open it. "+ + "Put the password the %s has into %s (0600, the value alone) and run again; then change it "+ + "on the server and accept the new value — this installer does not rotate a running %s", + check.what, check.volume, check.file, check.what, check.file, check.what) + } + return nil +} + // RootRewrite says what RewriteRoot did to the bundle. type RootRewrite struct { StoreURLs, BrokerURLs int @@ -162,16 +216,18 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@")) // The broker's administrator, changed once the broker answers and before anything dials it. - // Verified by a marker on the broker's own data volume, because the image carries nothing that - // can try a password from inside; what proves the password is the control plane answering - // over it, a few resources later. + // Verified by a marker on the broker's own data volume holding this password's fingerprint — + // the image carries nothing that can try a password from inside, and a marker that merely + // existed would let a regenerated password go unapplied for ever. What proves the password + // works is the control plane answering over it, a few resources later. + fp := credentialFingerprint(c.Broker) action := templateBrokerReady + "\n" + " {\n" + " \"id\": \"broker-admin\",\n" + " \"type\": \"action\",\n" + " \"in\": \"mesh-broker\",\n" + - " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && touch " + brokerAdminMarker + "\"],\n" + - " \"verify\": [\"sh\", \"-c\", \"test -f " + brokerAdminMarker + "\"]\n" + + " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && printf %s " + fp + " > " + brokerAdminMarker + "\"],\n" + + " \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" + " }," if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil { return out, err diff --git a/internal/bootstrap/talk.go b/internal/bootstrap/talk.go index 3b5b029..a9c4a8f 100644 --- a/internal/bootstrap/talk.go +++ b/internal/bootstrap/talk.go @@ -82,11 +82,9 @@ func (c controlPlane) carry(ctx context.Context, local, remote string) error { // landed unreadable, `secret accept` failed with `permission denied`, and what depended on it // crash-looped on material it never received. There is no shell in the image to chown it with. // -// What goes through here is a module manifest and a store connection string. The connection is the -// same value the produced bundle already holds in the clear — a foundation names its own bootstrap -// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The -// file on the machine is removed at once, and the copy inside the container goes when the -// container does, which for the temporary control plane is step 10. +// What goes through here is a module manifest — public, the same bytes as in the catalogue. A +// value that is secret goes through carryingSecret below. The file on the machine is removed at +// once, and the copy inside the container goes when the container does. func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error { local := filepath.Join(os.TempDir(), name) if err := os.WriteFile(local, content, 0o644); err != nil { @@ -96,6 +94,38 @@ func (c controlPlane) carrying(ctx context.Context, name string, content []byte, return c.carry(ctx, local, remote) } +// controlPlaneUID is the account the control plane's image runs as — `USER 65534:65534` in its +// Dockerfile — and so the only account inside the container that needs to read what is carried in. +const controlPlaneUID = 65534 + +// carryingSecret is carrying for a value that is a secret: staged in a directory only root can +// enter, at 0600, owned by the control plane's own account — which `docker cp` keeps, so inside +// the container the file is readable by the process that must read it and by nobody else. Since +// genesis makes the mesh's root credentials rather than copying the template's (rootsecrets.go), +// a store connection string or a broker password carried this way is a real secret, and 0644 in a +// shared temporary directory would hand it to any local user for the length of the copy. +func (c controlPlane) carryingSecret(ctx context.Context, name string, content []byte, remote string) error { + dir, err := os.MkdirTemp("", "mesh-carrying-") + if err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + defer os.RemoveAll(dir) + local := filepath.Join(dir, name) + if err := os.WriteFile(local, content, 0o600); err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + if err := os.Chown(local, controlPlaneUID, controlPlaneUID); err != nil { + // Not root — a test, or an installer run as a user, which no real genesis is. The + // directory is 0700, so nobody else on the machine can reach the file either way; inside + // the container the only account is the control plane's, so 0644 there is read by it and + // by nothing else. The narrower ownership is taken whenever it can be. + if err := os.Chmod(local, 0o644); err != nil { + return err + } + } + return c.carry(ctx, local, remote) +} + func indent(s string) string { if s == "" { return ""