package bootstrap import ( "bytes" "context" "crypto/rand" "encoding/base64" "fmt" "os" "path/filepath" "strings" "golang.org/x/crypto/bcrypt" "github.com/novox/mesh-host/internal/declaration" ) // The mesh's root credentials, made at genesis rather than copied from the template. // // **The template carries `bootstrap`, and a mesh raised from it kept it** (novox/hq issue 071). The // store's superuser and the controller's account on the bus are the two credentials every other one // rests on, and they were the two that were not secret: constants in a file anybody can read, // carried into the mesh by `secret accept` and marked as something the mesh must never replace — // which is correct for a credential that already created the databases, and made the well-known // value permanent. // // So the installer makes them. Two random values, **made once and kept on this machine** — the // store's at the path the postgres module declares as its own secret, so that when phase three // adopts the store `secret accept` carries in exactly the value the server was raised with; the // bus's beside the other things only genesis holds. A second run finds the files and changes // nothing, which is what lets the installer say "already done" about a store it must not restart. // // **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a // container's environment is in `docker inspect` for ever; the module that adopts the store // declares the same file mount, so the two specs are one and the applier reconciles rather than // recreates (phase3.go). // // **The bus has no administrator to change; it has a user list** (novox/hq ADR 0331 §3). The // bundle carries the first list — the controller's own account at `bootstrap` — and the // controller's address names the same account. Both are the bundle's placeholders: the address's // `nats://controller:bootstrap@` and the controller's entry in the carried list. The installer // puts the made password in the first and its bcrypt hash in the second, so the bus is raised with // the account the controller dials with. The controller records a hash of what it actually uses // (mesh-controller internal/broker, CredentialIn), so its own compositions carry the same account // onward, and the permanent controller is given it by `secret accept` (control.go deliverStores). const ( // StoreSuperuserFile is where the store's superuser password lives on the machine — the // postgres module's own-secret path, so genesis and adoption write the same file. StoreSuperuserFile = "/var/lib/postgres/superuser.secret" // StoreDataDir is where the store keeps its data — the postgres module's data directory, so // genesis raises the store where the module adopts it (novox/hq ADR 0338 §1). StoreDataDir = "/var/lib/mesh-store" // BusControllerFile is where the controller's password on the bus lives on the machine. Not a // module's path: no module reads it — the permanent controller is handed the address the // temporary one was given — and genesis keeps it only so a second run puts the same account // in the same two places. BusControllerFile = "/var/lib/mesh-genesis/bus-controller.secret" // BusControllerHashFile is the bcrypt hash of it, kept with it. bcrypt salts every hash, so a // hash made afresh on every run would rewrite the bus's user list on every run; kept, the // produced bundle is the same bundle twice. BusControllerHashFile = BusControllerFile + ".bcrypt" storeSuperuserMount = "/run/secrets/superuser" // What the template says, matched exactly. A template that says something else is a template // this installer does not know how to make safe, and it says so rather than guessing. templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` templateStoreVolumes = `"volumes": ["` + StoreDataDir + `:/var/lib/postgresql/data"]` templateStoreURL = "postgres:bootstrap@" // BusURLPlaceholder is the controller's account in its bus address, as the template names it. BusURLPlaceholder = "nats://" + busControllerUser + ":bootstrap@" // BusUserPlaceholder is the controller's entry in the user list the template carries, up to // its password's hash — written as it stands inside the bundle's JSON string, quotes escaped. BusUserPlaceholder = `{ user: \"` + busControllerUser + `\", password: \"` busControllerUser = "controller" ) // BusVar is where the control plane finds the mesh's bus, with its own account (mesh-controller // internal/broker NATSVar), and CertificateVar the certificate it pins the bus to (ADR 0004). const ( BusVar = "MESH_BUS_NATS" CertificateVar = "MESH_BROKER_CERTIFICATE" ) // BusPlaceholders refuses a template whose bus connection this installer does not recognise, // naming the placeholder it looked for (novox/hq ADR 0331 §3, §4). // // Asked at preflight, before anything is changed, and again where the bundle is rewritten. A // template with no `MESH_BUS_NATS` raises a control plane that cannot reach the mesh's bus and // says so every few seconds for ever (issue 146): the verify step would wait out its whole timeout // for an answer that cannot come. That is a fault in the bundle, and it is found here instead. func BusPlaceholders(template []byte, d *declaration.Declaration) error { control, err := controlPlaneIn(d) if err != nil { return err } address := strings.TrimSpace(control.Env[BusVar]) if address == "" { return fmt.Errorf("the template's control plane (%q) is given no %s, so it could not reach the mesh's "+ "bus and would never answer. A genesis template names the bus as %s
, which this "+ "installer fills with the account it makes; mesh-host's examples/foundation-first-node.lock "+ "is the shape", control.Name, BusVar, BusURLPlaceholder) } if !strings.HasPrefix(address, BusURLPlaceholder) { return fmt.Errorf("the template's control plane reaches the bus at %s, and this installer looked for "+ "%s
there: the account it makes goes where that placeholder is, and it will not "+ "guess where else a bus connection could be", redactURL(address), BusURLPlaceholder) } if strings.TrimSpace(control.Env[CertificateVar]) == "" { return fmt.Errorf("the template's control plane is given no %s, so it could not pin the bus's "+ "certificate (novox/hq ADR 0004) and would refuse to dial it", CertificateVar) } if n := bytes.Count(template, []byte(BusUserPlaceholder)); n != 1 { return fmt.Errorf("the template's user list for the bus names the controller's account %d times, and "+ "this installer looked for it once, as %s: the bus has to be raised with the account the "+ "controller dials with, and that is where it goes", n, BusUserPlaceholder) } return nil } // redactURL is a bus address with any password in it replaced, for an error that quotes it. func redactURL(address string) string { scheme, rest, has := strings.Cut(address, "://") if !has { rest, scheme = address, "" } at := strings.LastIndex(rest, "@") user, _, hasPassword := strings.Cut(rest[:max(at, 0)], ":") if at < 0 || !hasPassword { return address } if scheme != "" { scheme += "://" } return scheme + user + ":…" + rest[at:] } // RootCredentials are the two values, and whether this run made them. BusHash is the bcrypt hash of // Bus, as the bus's user list carries it. type RootCredentials struct { Store, Bus string BusHash string StoreMade, BusMade bool } // RootSecrets reads the credentials this machine already holds, or makes them. // // A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the // real one, and a machine that was only asked is not left holding half a genesis. func RootSecrets(dryRun bool) (RootCredentials, error) { return rootSecretsAt(StoreSuperuserFile, BusControllerFile, BusControllerHashFile, dryRun) } func rootSecretsAt(storeFile, busFile, hashFile string, dryRun bool) (RootCredentials, error) { var out RootCredentials var err error if out.Store, out.StoreMade, err = keptOrMade(storeFile, dryRun); err != nil { return out, err } if out.Bus, out.BusMade, err = keptOrMade(busFile, dryRun); err != nil { return out, err } if out.BusHash, err = keptHash(hashFile, out.Bus, dryRun); err != nil { return out, err } return out, nil } // keptHash is the bcrypt hash of password kept at path, made again only when the kept one is // missing or no longer the hash of this password. func keptHash(path, password string, dryRun bool) (string, error) { if kept, err := readCredentialFile(path); err == nil { if bcrypt.CompareHashAndPassword([]byte(kept), []byte(password)) == nil { return kept, nil } } else if !os.IsNotExist(err) { return "", err } hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { return "", err } if dryRun { return string(hash), nil } return string(hash), writeCredentialFile(path, string(hash)) } func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { value, err = readCredentialFile(path) if err == nil { return value, false, nil } if !os.IsNotExist(err) { return "", false, err } value, err = freshSecret() if err != nil { return "", false, err } if dryRun { return value, true, nil } return value, true, writeCredentialFile(path, value) } // writeCredentialFile writes a credential as genesis keeps it. func writeCredentialFile(path, value string) error { if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return err } // Written whole and renamed into place, at 0600, owned by whoever runs the installer — root, // which is also who the host runs as when it later writes the sealed copy here. // // **The value alone, no line ending.** The module that adopts the store declares this same // file, and what it declares is the value as `secret accept` took it — its line ending gone, // by design. The host folds a mounted file's content into the container's spec (novox/hq // 04-ISSUES/103), so a genesis that wrote `value\n` here would raise a store whose label // digests one byte more than the module's file, and phase three would RECREATE the store it // meant to adopt in place, with the temporary control plane connected to it. readCredentialFile // tolerates either ending, so a file an earlier genesis wrote still reads. tmp := path + ".genesis" if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil { return err } return os.Rename(tmp, path) } // readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone. // Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable". func readCredentialFile(path string) (string, error) { raw, err := os.ReadFile(path) if err != nil { return "", err } value := strings.TrimRight(string(raw), "\r\n") if value == "" { return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path) } return value, nil } // freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40 // characters, URL-safe — it lands inside connection strings. func freshSecret() (string, error) { b := make([]byte, 30) if _, err := rand.Read(b); err != nil { return "", err } return base64.RawURLEncoding.EncodeToString(b), nil } // Masking makes a reporter that never says the credentials this run made. // // The applier reports each action with its command line, and the context schemas' connection // strings carry a real password. Those lines go to a terminal and to whatever keeps the // transcript, which for the lab is a file. The exact values are known here, so they are replaced // wherever they appear, in every line said. func Masking(say func(string), c RootCredentials) func(string) { if c.Store == "" || c.Bus == "" { return say } replacer := strings.NewReplacer(c.Store, "…", c.Bus, "…") return func(line string) { say(replacer.Replace(line)) } } // RefuseExistingServers stops a run that would put a made credential in front of a store raised // by an earlier installer with the template's. // // The store's password is set by initdb, once, on an empty data directory. A machine that already holds // a cluster in the store's data directory and has no credential file was raised with `bootstrap`, and minting a new // value here would make a bundle that dials with a password the server does not have — failing // three steps later, in the schemas, with nothing pointing back here. Refused by name instead, // with the way forward. // // **The bus is not asked.** Its accounts live in the user list beside its configuration, which // the bundle writes and the bus re-reads, not in its data volume — so a bus raised earlier takes // the made account as readily as a new one. func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error { if !c.StoreMade { return nil } // Where the store keeps its data, as the postgres module does (novox/hq ADR 0338 §2): a cluster // there is one initdb already set a password for. The docker volume genesis used before is not // the store's data any more, and a machine holding only that is not refused. if _, err := run(ctx, "test", "-e", StoreDataDir+"/pgdata/PG_VERSION"); err != nil { return nil // no cluster there: a fresh machine, which is the case this installer makes } return fmt.Errorf( "this machine already holds the store's data (in "+StoreDataDir+") and no credential at %s, so it was raised "+ "by an earlier installer with the template's password. A new one made here would not open it. "+ "Put the password the store has into %s (0600, the value alone) and run again; then change it "+ "on the server and accept the new value — this installer does not rotate a running store", StoreSuperuserFile, StoreSuperuserFile) } // RootRewrite says what RewriteRoot did to the bundle. type RootRewrite struct { StoreURLs, BusURLs int } // RewriteRoot puts the made credentials into the produced bundle, in place of the template's. // // Byte for byte, like the image rewrite, so the file keeps its comments and a person can read what // was applied. Every replacement is counted and a count of zero is refused: a template that no // longer says what this expects is one whose credentials this would silently leave at the // well-known values, which is the fault this exists to remove. func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { var out RootRewrite bundle := r.Bundle // The store: a file, not an environment variable. var err error if bundle, err = replaceOnce(bundle, templateStorePassword, `"POSTGRES_PASSWORD_FILE": "`+storeSuperuserMount+`"`, "the store's password"); err != nil { return out, err } if bundle, err = replaceOnce(bundle, templateStoreVolumes, `"volumes": ["`+StoreDataDir+`:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, "the store's volumes"); err != nil { return out, err } // Everything that dials the store with the template's credentials. out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL)) if out.StoreURLs == 0 { return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL) } bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@")) // The bus: the controller's account, in its address and in the user list the bus is raised // with, the same account in both (novox/hq ADR 0331 §3). if err := BusPlaceholders(bundle, r.Declaration); err != nil { return out, err } out.BusURLs = bytes.Count(bundle, []byte(BusURLPlaceholder)) bundle = bytes.ReplaceAll(bundle, []byte(BusURLPlaceholder), []byte("nats://"+busControllerUser+":"+c.Bus+"@")) at := bytes.Index(bundle, []byte(BusUserPlaceholder)) + len(BusUserPlaceholder) end := bytes.Index(bundle[at:], []byte(`\"`)) if end < 0 { return out, fmt.Errorf("the controller's entry in the template's user list for the bus has no end to its password") } if !strings.HasPrefix(c.BusHash, "$2") { return out, fmt.Errorf("the controller's bus password was given no bcrypt hash, so the bus could not be raised with it; this is the installer's fault") } bundle = append(append(append([]byte{}, bundle[:at]...), c.BusHash...), bundle[at+end:]...) parsed, err := declaration.ParseFileTrusted(bundle) if err != nil { return out, fmt.Errorf("the bundle stopped being a declaration after its credentials were rewritten, which is this installer's fault: %w", err) } r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources) return out, nil } func replaceOnce(in []byte, from, to, what string) ([]byte, error) { switch n := bytes.Count(in, []byte(from)); n { case 1: return bytes.Replace(in, []byte(from), []byte(to), 1), nil case 0: return nil, fmt.Errorf("the template does not say %s the way this installer expects (%s), so it cannot be made safe here", what, from) default: return nil, fmt.Errorf("the template says %s %d times, and this installer expected once", what, n) } }