mesh-host/internal/bootstrap/rootsecrets.go

package bootstrap

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"golang.org/x/crypto/bcrypt"

	"github.com/novox/mesh-host/internal/declaration"
)

// The mesh's root credentials, made at genesis rather than copied from the template.
//
// **The template carries `bootstrap`, and a mesh raised from it kept it** (novox/hq issue 071). The
// store's superuser and the controller's account on the bus are the two credentials every other one
// rests on, and they were the two that were not secret: constants in a file anybody can read,
// carried into the mesh by `secret accept` and marked as something the mesh must never replace —
// which is correct for a credential that already created the databases, and made the well-known
// value permanent.
//
// So the installer makes them. Two random values, **made once and kept on this machine** — the
// store's at the path the postgres module declares as its own secret, so that when phase three
// adopts the store `secret accept` carries in exactly the value the server was raised with; the
// bus's beside the other things only genesis holds. A second run finds the files and changes
// nothing, which is what lets the installer say "already done" about a store it must not restart.
//
// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a
// container's environment is in `docker inspect` for ever; the module that adopts the store
// declares the same file mount, so the two specs are one and the applier reconciles rather than
// recreates (phase3.go).
//
// **The bus has no administrator to change; it has a user list** (novox/hq ADR 0331 §3). The
// bundle carries the first list — the controller's own account at `bootstrap` — and the
// controller's address names the same account. Both are the bundle's placeholders: the address's
// `nats://controller:bootstrap@` and the controller's entry in the carried list. The installer
// puts the made password in the first and its bcrypt hash in the second, so the bus is raised with
// the account the controller dials with. The controller records a hash of what it actually uses
// (mesh-controller internal/broker, CredentialIn), so its own compositions carry the same account
// onward, and the permanent controller is given it by `secret accept` (control.go deliverStores).
const (
	// StoreSuperuserFile is where the store's superuser password lives on the machine — the
	// postgres module's own-secret path, so genesis and adoption write the same file.
	StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
	// StoreDataDir is where the store keeps its data — the postgres module's data directory, so
	// genesis raises the store where the module adopts it (novox/hq ADR 0338 §1).
	StoreDataDir = "/var/lib/mesh-store"
	// BusControllerFile is where the controller's password on the bus lives on the machine. Not a
	// module's path: no module reads it — the permanent controller is handed the address the
	// temporary one was given — and genesis keeps it only so a second run puts the same account
	// in the same two places.
	BusControllerFile = "/var/lib/mesh-genesis/bus-controller.secret"
	// BusControllerHashFile is the bcrypt hash of it, kept with it. bcrypt salts every hash, so a
	// hash made afresh on every run would rewrite the bus's user list on every run; kept, the
	// produced bundle is the same bundle twice.
	BusControllerHashFile = BusControllerFile + ".bcrypt"

	storeSuperuserMount = "/run/secrets/superuser"

	// What the template says, matched exactly. A template that says something else is a template
	// this installer does not know how to make safe, and it says so rather than guessing.
	templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
	templateStoreVolumes  = `"volumes": ["` + StoreDataDir + `:/var/lib/postgresql/data"]`
	templateStoreURL      = "postgres:bootstrap@"

	// BusURLPlaceholder is the controller's account in its bus address, as the template names it.
	BusURLPlaceholder = "nats://" + busControllerUser + ":bootstrap@"
	// BusUserPlaceholder is the controller's entry in the user list the template carries, up to
	// its password's hash — written as it stands inside the bundle's JSON string, quotes escaped.
	BusUserPlaceholder = `{ user: \"` + busControllerUser + `\", password: \"`
	busControllerUser  = "controller"
)

// BusVar is where the control plane finds the mesh's bus, with its own account (mesh-controller
// internal/broker NATSVar), and CertificateVar the certificate it pins the bus to (ADR 0004).
const (
	BusVar         = "MESH_BUS_NATS"
	CertificateVar = "MESH_BROKER_CERTIFICATE"
)

// BusPlaceholders refuses a template whose bus connection this installer does not recognise,
// naming the placeholder it looked for (novox/hq ADR 0331 §3, §4).
//
// Asked at preflight, before anything is changed, and again where the bundle is rewritten. A
// template with no `MESH_BUS_NATS` raises a control plane that cannot reach the mesh's bus and
// says so every few seconds for ever (issue 146): the verify step would wait out its whole timeout
// for an answer that cannot come. That is a fault in the bundle, and it is found here instead.
func BusPlaceholders(template []byte, d *declaration.Declaration) error {
	control, err := controlPlaneIn(d)
	if err != nil {
		return err
	}
	address := strings.TrimSpace(control.Env[BusVar])
	if address == "" {
		return fmt.Errorf("the template's control plane (%q) is given no %s, so it could not reach the mesh's "+
			"bus and would never answer. A genesis template names the bus as %s<address>, which this "+
			"installer fills with the account it makes; mesh-host's examples/foundation-first-node.lock "+
			"is the shape", control.Name, BusVar, BusURLPlaceholder)
	}
	if !strings.HasPrefix(address, BusURLPlaceholder) {
		return fmt.Errorf("the template's control plane reaches the bus at %s, and this installer looked for "+
			"%s<address> there: the account it makes goes where that placeholder is, and it will not "+
			"guess where else a bus connection could be", redactURL(address), BusURLPlaceholder)
	}
	if strings.TrimSpace(control.Env[CertificateVar]) == "" {
		return fmt.Errorf("the template's control plane is given no %s, so it could not pin the bus's "+
			"certificate (novox/hq ADR 0004) and would refuse to dial it", CertificateVar)
	}
	if n := bytes.Count(template, []byte(BusUserPlaceholder)); n != 1 {
		return fmt.Errorf("the template's user list for the bus names the controller's account %d times, and "+
			"this installer looked for it once, as %s<hash>: the bus has to be raised with the account the "+
			"controller dials with, and that is where it goes", n, BusUserPlaceholder)
	}
	return nil
}

// redactURL is a bus address with any password in it replaced, for an error that quotes it.
func redactURL(address string) string {
	scheme, rest, has := strings.Cut(address, "://")
	if !has {
		rest, scheme = address, ""
	}
	at := strings.LastIndex(rest, "@")
	user, _, hasPassword := strings.Cut(rest[:max(at, 0)], ":")
	if at < 0 || !hasPassword {
		return address
	}
	if scheme != "" {
		scheme += "://"
	}
	return scheme + user + ":…" + rest[at:]
}

// RootCredentials are the two values, and whether this run made them. BusHash is the bcrypt hash of
// Bus, as the bus's user list carries it.
type RootCredentials struct {
	Store, Bus         string
	BusHash            string
	StoreMade, BusMade bool
}

// RootSecrets reads the credentials this machine already holds, or makes them.
//
// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the
// real one, and a machine that was only asked is not left holding half a genesis.
func RootSecrets(dryRun bool) (RootCredentials, error) {
	return rootSecretsAt(StoreSuperuserFile, BusControllerFile, BusControllerHashFile, dryRun)
}

func rootSecretsAt(storeFile, busFile, hashFile string, dryRun bool) (RootCredentials, error) {
	var out RootCredentials
	var err error
	if out.Store, out.StoreMade, err = keptOrMade(storeFile, dryRun); err != nil {
		return out, err
	}
	if out.Bus, out.BusMade, err = keptOrMade(busFile, dryRun); err != nil {
		return out, err
	}
	if out.BusHash, err = keptHash(hashFile, out.Bus, dryRun); err != nil {
		return out, err
	}
	return out, nil
}

// keptHash is the bcrypt hash of password kept at path, made again only when the kept one is
// missing or no longer the hash of this password.
func keptHash(path, password string, dryRun bool) (string, error) {
	if kept, err := readCredentialFile(path); err == nil {
		if bcrypt.CompareHashAndPassword([]byte(kept), []byte(password)) == nil {
			return kept, nil
		}
	} else if !os.IsNotExist(err) {
		return "", err
	}
	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		return "", err
	}
	if dryRun {
		return string(hash), nil
	}
	return string(hash), writeCredentialFile(path, string(hash))
}

func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
	value, err = readCredentialFile(path)
	if err == nil {
		return value, false, nil
	}
	if !os.IsNotExist(err) {
		return "", false, err
	}
	value, err = freshSecret()
	if err != nil {
		return "", false, err
	}
	if dryRun {
		return value, true, nil
	}
	return value, true, writeCredentialFile(path, value)
}

// writeCredentialFile writes a credential as genesis keeps it.
func writeCredentialFile(path, value string) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
		return err
	}
	// Written whole and renamed into place, at 0600, owned by whoever runs the installer — root,
	// which is also who the host runs as when it later writes the sealed copy here.
	//
	// **The value alone, no line ending.** The module that adopts the store declares this same
	// file, and what it declares is the value as `secret accept` took it — its line ending gone,
	// by design. The host folds a mounted file's content into the container's spec (novox/hq
	// 04-ISSUES/103), so a genesis that wrote `value\n` here would raise a store whose label
	// digests one byte more than the module's file, and phase three would RECREATE the store it
	// meant to adopt in place, with the temporary control plane connected to it. readCredentialFile
	// tolerates either ending, so a file an earlier genesis wrote still reads.
	tmp := path + ".genesis"
	if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
// Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable".
func readCredentialFile(path string) (string, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	value := strings.TrimRight(string(raw), "\r\n")
	if value == "" {
		return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path)
	}
	return value, nil
}

// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40
// characters, URL-safe — it lands inside connection strings.
func freshSecret() (string, error) {
	b := make([]byte, 30)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return base64.RawURLEncoding.EncodeToString(b), nil
}

// Masking makes a reporter that never says the credentials this run made.
//
// The applier reports each action with its command line, and the context schemas' connection
// strings carry a real password. Those lines go to a terminal and to whatever keeps the
// transcript, which for the lab is a file. The exact values are known here, so they are replaced
// wherever they appear, in every line said.
func Masking(say func(string), c RootCredentials) func(string) {
	if c.Store == "" || c.Bus == "" {
		return say
	}
	replacer := strings.NewReplacer(c.Store, "…", c.Bus, "…")
	return func(line string) { say(replacer.Replace(line)) }
}

// RefuseExistingServers stops a run that would put a made credential in front of a store raised
// by an earlier installer with the template's.
//
// The store's password is set by initdb, once, on an empty data directory. A machine that already holds
// a cluster in the store's data directory and has no credential file was raised with `bootstrap`, and minting a new
// value here would make a bundle that dials with a password the server does not have — failing
// three steps later, in the schemas, with nothing pointing back here. Refused by name instead,
// with the way forward.
//
// **The bus is not asked.** Its accounts live in the user list beside its configuration, which
// the bundle writes and the bus re-reads, not in its data volume — so a bus raised earlier takes
// the made account as readily as a new one.
func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error {
	if !c.StoreMade {
		return nil
	}
	// Where the store keeps its data, as the postgres module does (novox/hq ADR 0338 §2): a cluster
	// there is one initdb already set a password for. The docker volume genesis used before is not
	// the store's data any more, and a machine holding only that is not refused.
	if _, err := run(ctx, "test", "-e", StoreDataDir+"/pgdata/PG_VERSION"); err != nil {
		return nil // no cluster there: a fresh machine, which is the case this installer makes
	}
	return fmt.Errorf(
		"this machine already holds the store's data (in "+StoreDataDir+") and no credential at %s, so it was raised "+
			"by an earlier installer with the template's password. A new one made here would not open it. "+
			"Put the password the store has into %s (0600, the value alone) and run again; then change it "+
			"on the server and accept the new value — this installer does not rotate a running store",
		StoreSuperuserFile, StoreSuperuserFile)
}

// RootRewrite says what RewriteRoot did to the bundle.
type RootRewrite struct {
	StoreURLs, BusURLs int
}

// RewriteRoot puts the made credentials into the produced bundle, in place of the template's.
//
// Byte for byte, like the image rewrite, so the file keeps its comments and a person can read what
// was applied. Every replacement is counted and a count of zero is refused: a template that no
// longer says what this expects is one whose credentials this would silently leave at the
// well-known values, which is the fault this exists to remove.
func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
	var out RootRewrite
	bundle := r.Bundle

	// The store: a file, not an environment variable.
	var err error
	if bundle, err = replaceOnce(bundle, templateStorePassword,
		`"POSTGRES_PASSWORD_FILE": "`+storeSuperuserMount+`"`, "the store's password"); err != nil {
		return out, err
	}
	if bundle, err = replaceOnce(bundle, templateStoreVolumes,
		`"volumes": ["`+StoreDataDir+`:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`,
		"the store's volumes"); err != nil {
		return out, err
	}
	// Everything that dials the store with the template's credentials.
	out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL))
	if out.StoreURLs == 0 {
		return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL)
	}
	bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@"))

	// The bus: the controller's account, in its address and in the user list the bus is raised
	// with, the same account in both (novox/hq ADR 0331 §3).
	if err := BusPlaceholders(bundle, r.Declaration); err != nil {
		return out, err
	}
	out.BusURLs = bytes.Count(bundle, []byte(BusURLPlaceholder))
	bundle = bytes.ReplaceAll(bundle, []byte(BusURLPlaceholder),
		[]byte("nats://"+busControllerUser+":"+c.Bus+"@"))
	at := bytes.Index(bundle, []byte(BusUserPlaceholder)) + len(BusUserPlaceholder)
	end := bytes.Index(bundle[at:], []byte(`\"`))
	if end < 0 {
		return out, fmt.Errorf("the controller's entry in the template's user list for the bus has no end to its password")
	}
	if !strings.HasPrefix(c.BusHash, "$2") {
		return out, fmt.Errorf("the controller's bus password was given no bcrypt hash, so the bus could not be raised with it; this is the installer's fault")
	}
	bundle = append(append(append([]byte{}, bundle[:at]...), c.BusHash...), bundle[at+end:]...)

	parsed, err := declaration.ParseFileTrusted(bundle)
	if err != nil {
		return out, fmt.Errorf("the bundle stopped being a declaration after its credentials were rewritten, which is this installer's fault: %w", err)
	}
	r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources)
	return out, nil
}

func replaceOnce(in []byte, from, to, what string) ([]byte, error) {
	switch n := bytes.Count(in, []byte(from)); n {
	case 1:
		return bytes.Replace(in, []byte(from), []byte(to), 1), nil
	case 0:
		return nil, fmt.Errorf("the template does not say %s the way this installer expects (%s), so it cannot be made safe here", what, from)
	default:
		return nil, fmt.Errorf("the template says %s %d times, and this installer expected once", what, n)
	}
}