Installer review: secrets are staged privately, and a bundle is 0600 whether or not it existed

From review: the store and broker passwords genesis makes were carried into
the controller through a world-readable file in /tmp, a bundle left at 0644 by
an earlier installer kept that mode while now holding them, a mesh raised by
the old installer would have been handed new passwords its servers do not have,
and the broker-admin action's marker did not depend on the value. Secrets now
stage in a 0700 directory owned by the controller's account; the bundle is
chmod'd; an existing store or broker volume with no credential file is refused
by name; the marker holds the password's fingerprint. Also: one install path
for the store, broker and vault, no error-string matching for the operator
key, and no unreachable fallback for the superuser.

internal/bootstrap/rootsecrets.go

@@ -2,8 +2,11 @@ package bootstrap
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -80,12 +83,8 @@ func RootSecrets(dryRun bool) (RootCredentials, error) {
 }
 
 func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
-	raw, err := os.ReadFile(path)
+	value, err = readCredentialFile(path)
 	if err == nil {
-		value = strings.TrimRight(string(raw), "\r\n")
-		if value == "" {
-			return "", false, fmt.Errorf("%s exists and is empty; move it aside to have one made", path)
-		}
 		return value, false, nil
 	}
 	if !os.IsNotExist(err) {
@@ -113,6 +112,27 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
 	return value, true, nil
 }
 
+// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
+// Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable".
+func readCredentialFile(path string) (string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	value := strings.TrimRight(string(raw), "\r\n")
+	if value == "" {
+		return "", fmt.Errorf("%s exists and is empty; move it aside to have one made", path)
+	}
+	return value, nil
+}
+
+// credentialFingerprint names a credential without being one — what the broker-admin action
+// leaves on the broker's volume, so its verify holds for this value and not for any value.
+func credentialFingerprint(value string) string {
+	sum := sha256.Sum256([]byte(value))
+	return hex.EncodeToString(sum[:8])
+}
+
 // freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40
 // characters, URL-safe — it lands inside connection strings.
 func freshSecret() (string, error) {
@@ -123,6 +143,40 @@ func freshSecret() (string, error) {
 	return base64.RawURLEncoding.EncodeToString(b), nil
 }
 
+// RefuseExistingServers stops a run that would put a made credential in front of a server raised
+// by an earlier installer with the template's.
+//
+// The store's password is set by initdb, once, on an empty volume; the broker's by the action
+// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no
+// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a
+// bundle that dials with passwords the servers do not have — failing three steps later, in the
+// schemas, with nothing pointing back here. Refused by name instead, with the way forward.
+func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error {
+	for _, check := range []struct {
+		made   bool
+		volume string
+		what   string
+		file   string
+	}{
+		{c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile},
+		{c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile},
+	} {
+		if !check.made {
+			continue
+		}
+		if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil {
+			continue // no such volume: a fresh machine, which is the case this installer makes
+		}
+		return fmt.Errorf(
+			"this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+
+				"by an earlier installer with the template's password. A new one made here would not open it. "+
+				"Put the password the %s has into %s (0600, the value alone) and run again; then change it "+
+				"on the server and accept the new value — this installer does not rotate a running %s",
+			check.what, check.volume, check.file, check.what, check.file, check.what)
+	}
+	return nil
+}
+
 // RootRewrite says what RewriteRoot did to the bundle.
 type RootRewrite struct {
 	StoreURLs, BrokerURLs int
@@ -162,16 +216,18 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
 	bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@"))
 
 	// The broker's administrator, changed once the broker answers and before anything dials it.
-	// Verified by a marker on the broker's own data volume, because the image carries nothing that
-	// can try a password from inside; what proves the password is the control plane answering
-	// over it, a few resources later.
+	// Verified by a marker on the broker's own data volume holding this password's fingerprint —
+	// the image carries nothing that can try a password from inside, and a marker that merely
+	// existed would let a regenerated password go unapplied for ever. What proves the password
+	// works is the control plane answering over it, a few resources later.
+	fp := credentialFingerprint(c.Broker)
 	action := templateBrokerReady + "\n" +
 		"    {\n" +
 		"      \"id\": \"broker-admin\",\n" +
 		"      \"type\": \"action\",\n" +
 		"      \"in\": \"mesh-broker\",\n" +
-		"      \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && touch " + brokerAdminMarker + "\"],\n" +
-		"      \"verify\": [\"sh\", \"-c\", \"test -f " + brokerAdminMarker + "\"]\n" +
+		"      \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && printf %s " + fp + " > " + brokerAdminMarker + "\"],\n" +
+		"      \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" +
 		"    },"
 	if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil {
 		return out, err