diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a640b5f..049112a 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -48,6 +48,7 @@ const ( StepApply Step = "apply" StepVerify Step = "verify" StepEnrol Step = "enrol" + StepOperator Step = "operator" StepRegistry Step = "registry" StepPublish Step = "publish" StepControlPlane Step = "control-plane" @@ -57,10 +58,13 @@ const ( StepSDK Step = "sdk" StepBase Step = "base" StepStore Step = "store" + StepBroker Step = "broker" + StepVault Step = "vault" StepCatalogue Step = "catalogue" StepNetwork Step = "network" StepFilter Step = "filter" StepExtras Step = "extras" + StepExport Step = "export" ) // Steps in the order they happen, so a failure can say "step 2 of 11". @@ -76,12 +80,12 @@ const ( // mesh made, out of a repository and a commit it can name, and can therefore make again. var Steps = []Step{ StepPreflight, StepLoad, StepBuild, StepBundle, StepApply, StepVerify, - StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder, + StepEnrol, StepOperator, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder, StepPackages, StepSDK, // Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to // build, to say what it holds, on its network, filtering. They used to be things somebody // typed afterwards, which is how they went missing without anything complaining. - StepBase, StepStore, StepCatalogue, StepNetwork, StepFilter, StepExtras, + StepBase, StepStore, StepBroker, StepVault, StepCatalogue, StepNetwork, StepFilter, StepExtras, StepExport, } // Error is a failure, named by the step it happened in. @@ -201,8 +205,13 @@ type Deps struct { // Result is what the bootstrap did, in the shape `--json` prints. type Result struct { - System string `json:"system"` - DryRun bool `json:"dry-run,omitempty"` + // OperatorKey is where the operator's private key was written; OperatorKeyMade whether this + // run made it. RootExport is where the operator-sealed export landed. + OperatorKey string `json:"operator-key,omitempty"` + OperatorKeyMade bool `json:"operator-key-made,omitempty"` + RootExport string `json:"root-export,omitempty"` + System string `json:"system"` + DryRun bool `json:"dry-run,omitempty"` // Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and // what the produced bundle names it by. @@ -380,6 +389,36 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro } result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out result.Temporary = rewritten.TempName + // The mesh's root credentials: made here, never the template's (novox/hq issue 071). + creds, err := RootSecrets(o.DryRun) + if err != nil { + return result, failed(StepBundle, err) + } + if err := RefuseExistingServers(ctx, d.Run, creds); err != nil { + return result, failed(StepBundle, err) + } + // From here on nothing this installer says contains the values it just made. + say = Masking(say, creds) + root, err := RewriteRoot(&rewritten, creds) + if err != nil { + return result, failed(StepBundle, err) + } + for _, c := range []struct { + what, path string + made bool + }{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} { + switch { + case c.made && o.DryRun: + say(fmt.Sprintf(" %-17s would be made and kept at %s (0600)", c.what, c.path)) + case c.made: + say(fmt.Sprintf(" %-17s made, kept at %s (0600)", c.what, c.path)) + default: + say(fmt.Sprintf(" %-17s already at %s — kept", c.what, c.path)) + } + } + say(fmt.Sprintf(" credentials the template's bootstrap and guest are gone: %d store and %d broker "+ + "connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers", + root.StoreURLs, root.BrokerURLs)) if rewritten.Renamed { say(fmt.Sprintf(" control plane %s, renamed from %s", rewritten.TempName, rewritten.WasCalled)) @@ -509,6 +548,18 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepEnrol, err) } + // ---- operator — the key the mesh's root secrets are also sealed to, held by a person ------ + // + // Before anything is accepted into the mesh: the store's and broker's credentials go in during + // the control-plane step, and they must be sealed to this key as well as to the node, or they + // are as unrecoverable as the constants they replaced (novox/hq ADR 0085, amended). + say("operator — a key the mesh seals its root secrets to, held by a person and never by the mesh") + operator, err := MakeOperatorKey(ctx, o, temporary, say) + result.OperatorKey, result.OperatorKeyMade = operator.Path, operator.Made + if err != nil { + return result, failed(StepOperator, err) + } + // ---- 7. registry ---------------------------------------------------------------------- say("registry — somewhere for this mesh to keep its own images") registry, err := InstallRegistry(ctx, o, d, temporary, say) @@ -598,6 +649,20 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepStore, err) } + // ---- broker --------------------------------------------------------------------------- + say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two") + if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { + return result, failed(StepBroker, err) + } + + // ---- vault ---------------------------------------------------------------------------- + // A foundation module (novox/hq ADR 0085, amended): it keeps every operator-sealed secret on + // its own disk, outside the store, from the first push that carries one. + say("vault — the mesh's secrets get an owner, and their operator-sealed copies a place to live") + if err := InstallVault(ctx, o, permanentControl, say); err != nil { + return result, failed(StepVault, err) + } + // ---- 15. catalogue -------------------------------------------------------------------- say("catalogue — the module graph: what is held, what a change reaches, what to rebuild") if err := InstallFromCatalogue(ctx, o, permanentControl, "mesh-catalog", say); err != nil { @@ -622,6 +687,14 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepExtras, err) } + // ---- export — what the operator keeps beside the key --------------------------------- + say("export — every root secret, sealed to the operator key, written beside it") + exported, err := ExportRootSecrets(ctx, o, permanentControl, say) + result.RootExport = exported + if err != nil { + return result, failed(StepExport, err) + } + say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " + "sits on its private network, and filters what modules declared.") say("what remains is somebody else's: adding nodes, and assigning what they should run.") diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go index 3125dd0..e806a44 100644 --- a/internal/bootstrap/control.go +++ b/internal/bootstrap/control.go @@ -274,7 +274,7 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. at := "/accepting-" + secret - if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { + if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, diff --git a/internal/bootstrap/operator.go b/internal/bootstrap/operator.go new file mode 100644 index 0000000..761689c --- /dev/null +++ b/internal/bootstrap/operator.go @@ -0,0 +1,100 @@ +package bootstrap + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/identity" +) + +// The operator's sealing key: made at genesis, before the mesh is told any secret. +// +// Every secret a module holds for itself is sealed to the node that uses it; from here on it is +// sealed to this key as well, and the vault keeps those copies (novox/hq ADR 0085, amended). The +// private half is written once, beside the produced bundle, and given to nothing: the mesh +// records the public half and can open nothing it seals to it. The operator copies the file off +// the machine and keeps it — it is what recovers the mesh's root secrets when a node cannot. +// +// **Before enrolment's first `secret accept`**, or the credentials genesis made would be sealed +// to the node alone and be exactly as unrecoverable as the constants they replaced. + +// OperatorKeyFile is where the private half is written, beside the bundle. +func OperatorKeyFile(o Options) string { + return filepath.Join(filepath.Dir(o.Out), "operator.key") +} + +// RootExportFile is where the export of every operator-sealed secret is written at the end. +func RootExportFile(o Options) string { + return filepath.Join(filepath.Dir(o.Out), "root-secrets.export.json") +} + +type OperatorKey struct { + Path string + Fingerprint string + Made bool +} + +// MakeOperatorKey makes the key if this machine has none, and tells the mesh its public half. +func MakeOperatorKey(ctx context.Context, o Options, control controlPlane, say func(string)) (OperatorKey, error) { + out := OperatorKey{Path: OperatorKeyFile(o)} + var key identity.SealingKey + if _, err := os.Stat(out.Path); err == nil { + key, err = identity.LoadSealingKey(out.Path) + if err != nil { + return out, err + } + say(" operator key already at " + out.Path + " — kept") + } else if os.IsNotExist(err) { + key, err = identity.GenerateSealingKey() + if err != nil { + return out, err + } + if err := os.MkdirAll(filepath.Dir(out.Path), 0o755); err != nil { + return out, err + } + if err := os.WriteFile(out.Path, []byte(key.Private+"\n"), 0o600); err != nil { + return out, err + } + out.Made = true + } else { + return out, err + } + sum := sha256.Sum256([]byte(key.Public)) + out.Fingerprint = "sha256:" + hex.EncodeToString(sum[:8]) + + if _, err := control.tell(ctx, "operator", "key", "set", key.Public); err != nil { + return out, err + } + if out.Made { + say(" operator key " + out.Fingerprint + " — private half at " + out.Path + " (0600)") + say(" COPY IT OFF THIS MACHINE AND KEEP IT: it opens the mesh's root secrets, and") + say(" nothing else does. The mesh holds only the public half.") + } else { + say(" operator key " + out.Fingerprint + " — the mesh seals its root secrets to it") + } + return out, nil +} + +// ExportRootSecrets writes the export beside the operator key: every secret sealed to it, as +// ciphertext, and the honest list of what is not. What the vault keeps on its disk, kept once +// more by the person who holds the key. +func ExportRootSecrets(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + path := RootExportFile(o) + body, err := control.tell(ctx, "secret", "export") + if err != nil { + return path, err + } + if !strings.Contains(body, `"kept"`) { + return path, fmt.Errorf("`secret export` did not produce an export:\n%s", body) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + return path, err + } + say(" exported " + path + " (0600) — ciphertext, sealed to the operator key; keep it with the key") + return path, nil +} diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index a74ce76..e39b9e3 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -72,6 +72,27 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, } say(" adopting " + store.Name + " — the store the foundation raised, unchanged") + // The superuser is the foundation's, made at genesis — carried in before the push, or the push + // would seal random bytes where a working password has to be and the provisioner would not open + // the store it is meant to manage. + if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", + func() error { + return deliverCredential(ctx, o, control, module, "superuser", StoreSuperuserFile, "the store's superuser", say) + }, + say); err != nil { + return err + } + say(" adopted mesh-store — the foundation's store is now the " + module + " module") + return nil +} + +// installProvider registers, builds, issues, assigns and pushes one catalogue module, with one +// thing done just before the push — the moment a credential the mesh could not have made has to +// be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker +// and the vault each need it or need the shape, and three copies of it drifted. +func installProvider(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, buildNote string, beforePush func() error, say func(string)) error { + remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { return err @@ -86,7 +107,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, "the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where a builder "+ "clones it", module) } - say(" building " + module + " (the provisioner; the server is adopted, not built)") + say(strings.TrimRight(" building "+module+" "+buildNote, " ")) if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { return err @@ -101,18 +122,80 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } + if beforePush != nil { + if err := beforePush(); err != nil { + return err + } + } + _, err := pushNode(ctx, o, control, say) + return err +} - // The superuser is the foundation's, made at genesis — carried in before the push, or the push - // would seal random bytes where a working password has to be and the provisioner would not open - // the store it is meant to manage. - if err := deliverSuperuser(ctx, o, control, module, store, say); err != nil { +// deliverCredential carries a credential genesis made into a module as its own secret, through +// `secret accept`: the mesh cannot invent the value a running server already has. +func deliverCredential(ctx context.Context, o Options, control controlPlane, module, secret, file, what string, + say func(string)) error { + + value, err := readCredentialFile(file) + if err != nil { + return fmt.Errorf("%s is not at %s, so the %s module has nothing to open its server with — "+ + "and the mesh cannot invent the one the server already has: %w", what, file, module, err) + } + at := "/accepting-" + secret + if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return err } - - if _, err := pushNode(ctx, o, control, say); err != nil { + if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { return err } - say(" adopted mesh-store — the foundation's store is now the " + module + " module") + say(" accepted " + secret + " — " + what + ", as genesis made it") + return nil +} + +// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same +// shape as InstallStore, for the same reasons. The administrator's password is the one genesis +// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can +// reach the management API as it. +func InstallBroker(ctx context.Context, o Options, control controlPlane, + foundation *declaration.Declaration, say func(string)) error { + + const module = "lavinmq" + manifest, err := readManifest(o.Catalogue, module) + if err != nil { + return err + } + broker, err := brokerIn(foundation) + if err != nil { + return err + } + if err := serverMatchesFoundation(manifest, broker, module); err != nil { + return err + } + say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged") + if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", + func() error { + return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say) + }, + say); err != nil { + return err + } + say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module") + return nil +} + +// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to +// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider, +// and from its first push it keeps the export of every operator-sealed secret on its own disk. +func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error { + const module = "mesh-vault" + manifest, err := readManifest(o.Catalogue, module) + if err != nil { + return err + } + if err := installProvider(ctx, o, control, module, manifest, "", nil, say); err != nil { + return err + } + say(" installed " + module + " — the operator-sealed export now lives on this machine, outside the store") return nil } @@ -176,30 +259,3 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu "store with. Its server container has to carry the name the foundation raised", module, store.Name) } - -// deliverSuperuser carries the store's superuser password into the module. -// -// It is the foundation's, set on the bundle's store container at genesis; the mesh cannot invent a -// credential that already made the databases, so it goes in through `secret accept`, exactly as the -// control plane's store connections do (control.go deliverStores). -func deliverSuperuser(ctx context.Context, o Options, control controlPlane, module string, - store *declaration.Container, say func(string)) error { - - const secret = "superuser" - value := strings.TrimSpace(store.Env["POSTGRES_PASSWORD"]) - if value == "" { - return fmt.Errorf( - "the foundation's store names no POSTGRES_PASSWORD, so the %s module has no superuser "+ - "to open it with — and the mesh cannot invent the one that already made the databases", - module) - } - at := "/accepting-" + secret - if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { - return err - } - if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { - return err - } - say(" accepted " + secret + " — the store's superuser, as the foundation made it") - return nil -} diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index d7f24ca..1a4ceed 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -270,7 +270,7 @@ func packagePasswords() (db, admin, builder string, err error) { func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string, say func(string)) error { at := "/accepting-npm-password" - if err := control.carrying(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { + if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { return err } if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil { diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go index 45eb470..456b219 100644 --- a/internal/bootstrap/rewrite.go +++ b/internal/bootstrap/rewrite.go @@ -316,21 +316,26 @@ func sortStrings(values []string) { // writeBundleFile puts the produced bundle where a person can read it, creating the directory it // lives in. // -// 0644, and that is deliberate: this file names an image and describes a foundation, and it holds -// the bootstrap credentials the template happens to carry — which are the same ones anybody can -// read in the template itself. It is meant to be read. What must not be world-readable is the -// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`). +// 0600: the produced bundle carries the credentials genesis made — the store's and the broker's, +// inside the temporary control plane's connection strings (novox/hq issue 071). It used to be +// 0644 and say so was fine because the template's credentials were the same ones anybody could +// read in the template; they are not any more. Still meant to be read, by root. func writeBundleFile(path string, content []byte) error { if dir := filepath.Dir(path); dir != "" && dir != "." { if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err) } } - if err := os.WriteFile(path, content, 0o644); err != nil { + if err := os.WriteFile(path, content, 0o600); err != nil { return fmt.Errorf( "cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+ "applying something nobody can read afterwards is how a machine becomes a mystery", path, err) } + // The mode above applies only when the file is created. A bundle an earlier installer left at + // 0644 would keep that while now carrying real credentials, with this function saying 0600. + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("cannot make the produced bundle %s readable by root alone: %w", path, err) + } return nil } diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go new file mode 100644 index 0000000..06a81f2 --- /dev/null +++ b/internal/bootstrap/rootsecrets.go @@ -0,0 +1,269 @@ +package bootstrap + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// The mesh's root credentials, made at genesis rather than copied from the template. +// +// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq +// issue 071). The store's superuser and the broker's administrator are the two credentials every +// other one rests on, and they were the two that were not secret: constants in a file anybody can +// read, carried into the mesh by `secret accept` and marked as something the mesh must never +// replace — which is correct for a credential that already created the databases, and made the +// well-known value permanent. +// +// So the installer makes them. Two random values, **made once and kept on this machine** at the +// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three +// adopts the store and the broker, `secret accept` carries in exactly the value the servers were +// raised with, and the host's later write of the sealed secret lands the same bytes in the same +// file. A second run finds the files and changes nothing, which is what lets the installer say +// "already done" about a store it must not restart. +// +// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a +// container's environment is in `docker inspect` for ever; the module that adopts the store +// declares the same file mount, so the two specs are one and the applier reconciles rather than +// recreates (phase3.go). The broker has no such file: its image's default administrator is changed +// in place by an action once the broker answers, and the produced bundle carries that action. +const ( + // StoreSuperuserFile is where the store's superuser password lives on the machine — the + // postgres module's own-secret path, so genesis and adoption write the same file. + StoreSuperuserFile = "/var/lib/postgres/superuser.secret" + // BrokerAdminFile is the same for the broker's administrator — the lavinmq module's. + BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret" + // BrokerAdminUser is the broker's administrator. The image's default account, kept by name + // and given a password that is not the image's default; a renamed account would have to be + // created before anything can authenticate, and the thing that creates accounts is the thing + // that has to authenticate first. + BrokerAdminUser = "guest" + + storeSuperuserMount = "/run/secrets/superuser" + + // What the template says, matched exactly. A template that says something else is a template + // this installer does not know how to make safe, and it says so rather than guessing. + templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` + templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]` + templateStoreURL = "postgres:bootstrap@" + templateBrokerURL = "guest:guest@" + templateBrokerReady = "\"verify\": [\"lavinmqctl\", \"status\"]\n }," + brokerAdminMarker = "/var/lib/lavinmq/.mesh-admin" +) + +// RootCredentials are the two values, and whether this run made them. +type RootCredentials struct { + Store, Broker string + StoreMade, BrokerMade bool +} + +// RootSecrets reads the credentials this machine already holds, or makes them. +// +// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the +// real one, and a machine that was only asked is not left holding half a genesis. +func RootSecrets(dryRun bool) (RootCredentials, error) { + var out RootCredentials + var err error + if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil { + return out, err + } + if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil { + return out, err + } + return out, nil +} + +func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { + value, err = readCredentialFile(path) + if err == nil { + return value, false, nil + } + if !os.IsNotExist(err) { + return "", false, err + } + value, err = freshSecret() + if err != nil { + return "", false, err + } + if dryRun { + return value, true, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", false, err + } + // Written whole and renamed into place, at 0600, owned by whoever runs the installer — root, + // which is also who the host runs as when it later writes the sealed copy here. + tmp := path + ".genesis" + if err := os.WriteFile(tmp, []byte(value+"\n"), 0o600); err != nil { + return "", false, err + } + if err := os.Rename(tmp, path); err != nil { + return "", false, err + } + return value, true, nil +} + +// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone. +// Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable". +func readCredentialFile(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimRight(string(raw), "\r\n") + if value == "" { + return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path) + } + return value, nil +} + +// credentialFingerprint names a credential without being one — what the broker-admin action +// leaves on the broker's volume, so its verify holds for this value and not for any value. +func credentialFingerprint(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:8]) +} + +// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40 +// characters, URL-safe — it lands inside connection strings. +func freshSecret() (string, error) { + b := make([]byte, 30) + if _, err := rand.Read(b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +// Masking makes a reporter that never says the credentials this run made. +// +// The applier reports each action with its command line, and two of them now carry a real +// password — the context schemas' connection strings and the broker's change_password. Those +// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The +// exact values are known here, so they are replaced wherever they appear, in every line said. +func Masking(say func(string), c RootCredentials) func(string) { + replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…") + if c.Store == "" || c.Broker == "" { + return say + } + return func(line string) { say(replacer.Replace(line)) } +} + +// RefuseExistingServers stops a run that would put a made credential in front of a server raised +// by an earlier installer with the template's. +// +// The store's password is set by initdb, once, on an empty volume; the broker's by the action +// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no +// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a +// bundle that dials with passwords the servers do not have — failing three steps later, in the +// schemas, with nothing pointing back here. Refused by name instead, with the way forward. +func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error { + for _, check := range []struct { + made bool + volume string + what string + file string + }{ + {c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile}, + {c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile}, + } { + if !check.made { + continue + } + if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil { + continue // no such volume: a fresh machine, which is the case this installer makes + } + return fmt.Errorf( + "this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+ + "by an earlier installer with the template's password. A new one made here would not open it. "+ + "Put the password the %s has into %s (0600, the value alone) and run again; then change it "+ + "on the server and accept the new value — this installer does not rotate a running %s", + check.what, check.volume, check.file, check.what, check.file, check.what) + } + return nil +} + +// RootRewrite says what RewriteRoot did to the bundle. +type RootRewrite struct { + StoreURLs, BrokerURLs int +} + +// RewriteRoot puts the made credentials into the produced bundle, in place of the template's. +// +// Byte for byte, like the image rewrite, so the file keeps its comments and a person can read what +// was applied. Every replacement is counted and a count of zero is refused: a template that no +// longer says what this expects is one whose credentials this would silently leave at the +// well-known values, which is the fault this exists to remove. +func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { + var out RootRewrite + bundle := r.Bundle + + // The store: a file, not an environment variable. + var err error + if bundle, err = replaceOnce(bundle, templateStorePassword, + `"POSTGRES_PASSWORD_FILE": "`+storeSuperuserMount+`"`, "the store's password"); err != nil { + return out, err + } + if bundle, err = replaceOnce(bundle, templateStoreVolumes, + `"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, + "the store's volumes"); err != nil { + return out, err + } + // Everything that dials the store or the broker with the template's credentials. + out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL)) + if out.StoreURLs == 0 { + return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL) + } + bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@")) + out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL)) + if out.BrokerURLs == 0 { + return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL) + } + bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@")) + + // The broker's administrator, changed once the broker answers and before anything dials it. + // Verified by a marker on the broker's own data volume holding this password's fingerprint — + // the image carries nothing that can try a password from inside, and a marker that merely + // existed would let a regenerated password go unapplied for ever. What proves the password + // works is the control plane answering over it, a few resources later. The marker ends in a + // newline because the verify reads it with the shell's `read`, which fails at end of file + // without one — an action that ran and a verify that said no, once, in the lab. + fp := credentialFingerprint(c.Broker) + action := templateBrokerReady + "\n" + + " {\n" + + " \"id\": \"broker-admin\",\n" + + " \"type\": \"action\",\n" + + " \"in\": \"mesh-broker\",\n" + + " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" + + " \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" + + " }," + if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil { + return out, err + } + + parsed, err := declaration.ParseFileTrusted(bundle) + if err != nil { + return out, fmt.Errorf("the bundle stopped being a declaration after its credentials were rewritten, which is this installer's fault: %w", err) + } + r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources) + return out, nil +} + +func replaceOnce(in []byte, from, to, what string) ([]byte, error) { + switch n := bytes.Count(in, []byte(from)); n { + case 1: + return bytes.Replace(in, []byte(from), []byte(to), 1), nil + case 0: + return nil, fmt.Errorf("the template does not say %s the way this installer expects (%s), so it cannot be made safe here", what, from) + default: + return nil, fmt.Errorf("the template says %s %d times, and this installer expected once", what, n) + } +} diff --git a/internal/bootstrap/rootsecrets_test.go b/internal/bootstrap/rootsecrets_test.go new file mode 100644 index 0000000..c23f864 --- /dev/null +++ b/internal/bootstrap/rootsecrets_test.go @@ -0,0 +1,167 @@ +package bootstrap + +import ( + "os" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// The produced bundle carries no well-known credential: the store reads its password from the +// file genesis made, every connection string names the made values, and the broker's default +// administrator is changed by an action before anything dials it (novox/hq issue 071). +func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"} + got, err := RewriteRoot(&r, creds) + if err != nil { + t.Fatal(err) + } + text := string(r.Bundle) + for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} { + if strings.Contains(text, gone) { + t.Errorf("the produced bundle still says %s", gone) + } + } + if got.StoreURLs < 3 || got.BrokerURLs < 2 { + t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs) + } + var store, action bool + for _, res := range r.Declaration.Resources { + switch x := res.(type) { + case *declaration.Container: + if x.Name != "mesh-store" { + continue + } + store = true + if _, has := x.Env["POSTGRES_PASSWORD"]; has { + t.Error("the store still takes its password from its environment") + } + if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount { + t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"]) + } + if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) { + t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes) + } + case *declaration.Action: + if x.ID != "broker-admin" { + continue + } + action = true + if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") { + t.Errorf("the broker-admin action is %v in %q", x.Command, x.In) + } + } + } + if !store || !action { + t.Fatalf("store=%v action=%v", store, action) + } + // The order matters: the broker's password changes after it answers and before the control + // plane, which dials it with the new one, is raised. + var readyAt, adminAt, controlAt int + for i, res := range r.Declaration.Resources { + switch res.Identity() { + case "broker-ready": + readyAt = i + case "broker-admin": + adminAt = i + case "control-plane": + controlAt = i + } + } + if !(readyAt < adminAt && adminAt < controlAt) { + t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt) + } +} + +// A template that no longer says what this expects is refused, not half-rewritten. +func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) { + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1) + r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil { + t.Fatal("a template with an unknown store password was rewritten") + } +} + +// Made once and kept: a second run reads the same value; a dry run writes nothing. +func TestRootSecretsAreKeptAcrossRuns(t *testing.T) { + dir := t.TempDir() + path := dir + "/superuser.secret" + first, made, err := keptOrMade(path, false) + if err != nil || !made || len(first) != 40 { + t.Fatalf("first: %q made=%v err=%v", first, made, err) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Errorf("mode %v", info.Mode().Perm()) + } + second, made, err := keptOrMade(path, false) + if err != nil || made || second != first { + t.Fatalf("second: %q made=%v err=%v", second, made, err) + } + dry := dir + "/dry.secret" + if _, made, err := keptOrMade(dry, true); err != nil || !made { + t.Fatal(err) + } + if _, err := os.Stat(dry); err == nil { + t.Fatal("a dry run wrote a secret") + } +} + +// Nothing the installer says after making the credentials contains them. +func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) { + var said []string + say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"}) + say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)") + say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)") + for _, l := range said { + if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") { + t.Errorf("said a credential: %s", l) + } + } + if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") { + t.Errorf("the lines were not the same lines with the values masked: %v", said) + } +} + +// The broker-admin marker is written with a line ending, because the verify reads it with `read`. +func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) { + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { + t.Fatal(err) + } + for _, res := range r.Declaration.Resources { + a, ok := res.(*declaration.Action) + if !ok || a.ID != "broker-admin" { + continue + } + cmd := strings.Join(a.Command, " ") + if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") { + t.Errorf("the marker is written without a line ending: %s", cmd) + } + if !strings.Contains(strings.Join(a.Verify, " "), "read m <") { + t.Errorf("the verify does not read the marker: %v", a.Verify) + } + } +} diff --git a/internal/bootstrap/talk.go b/internal/bootstrap/talk.go index 3b5b029..a9c4a8f 100644 --- a/internal/bootstrap/talk.go +++ b/internal/bootstrap/talk.go @@ -82,11 +82,9 @@ func (c controlPlane) carry(ctx context.Context, local, remote string) error { // landed unreadable, `secret accept` failed with `permission denied`, and what depended on it // crash-looped on material it never received. There is no shell in the image to chown it with. // -// What goes through here is a module manifest and a store connection string. The connection is the -// same value the produced bundle already holds in the clear — a foundation names its own bootstrap -// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The -// file on the machine is removed at once, and the copy inside the container goes when the -// container does, which for the temporary control plane is step 10. +// What goes through here is a module manifest — public, the same bytes as in the catalogue. A +// value that is secret goes through carryingSecret below. The file on the machine is removed at +// once, and the copy inside the container goes when the container does. func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error { local := filepath.Join(os.TempDir(), name) if err := os.WriteFile(local, content, 0o644); err != nil { @@ -96,6 +94,38 @@ func (c controlPlane) carrying(ctx context.Context, name string, content []byte, return c.carry(ctx, local, remote) } +// controlPlaneUID is the account the control plane's image runs as — `USER 65534:65534` in its +// Dockerfile — and so the only account inside the container that needs to read what is carried in. +const controlPlaneUID = 65534 + +// carryingSecret is carrying for a value that is a secret: staged in a directory only root can +// enter, at 0600, owned by the control plane's own account — which `docker cp` keeps, so inside +// the container the file is readable by the process that must read it and by nobody else. Since +// genesis makes the mesh's root credentials rather than copying the template's (rootsecrets.go), +// a store connection string or a broker password carried this way is a real secret, and 0644 in a +// shared temporary directory would hand it to any local user for the length of the copy. +func (c controlPlane) carryingSecret(ctx context.Context, name string, content []byte, remote string) error { + dir, err := os.MkdirTemp("", "mesh-carrying-") + if err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + defer os.RemoveAll(dir) + local := filepath.Join(dir, name) + if err := os.WriteFile(local, content, 0o600); err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + if err := os.Chown(local, controlPlaneUID, controlPlaneUID); err != nil { + // Not root — a test, or an installer run as a user, which no real genesis is. The + // directory is 0700, so nobody else on the machine can reach the file either way; inside + // the container the only account is the control plane's, so 0644 there is read by it and + // by nothing else. The narrower ownership is taken whenever it can be. + if err := os.Chmod(local, 0o644); err != nil { + return err + } + } + return c.carry(ctx, local, remote) +} + func indent(s string) string { if s == "" { return ""