A node's serving key is stored in the format a server reads

PKCS#8 PEM, not this host's own base64. The mesh delivers a PEM certificate
beside it and every TLS server there is reads PEM: nginx's ssl_certificate_key,
Go's LoadX509KeyPair, openssl s_server. Stored the other way the file was
intact, present, correctly permissioned, and unusable — the machine failed at
the moment something connected, which the lab found by connecting.

A key in the old encoding is refused by name rather than called corrupt: it is
replaced by enrolling again, and that is a different remedy from a damaged
file.

internal/identity/serving.go

@@ -3,7 +3,9 @@ package identity
 import (
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/x509"
 	"encoding/base64"
+	"encoding/pem"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -51,6 +53,12 @@ func GenerateServingKey() (ServingKey, error) {
 // A file of its own, named by whatever configuration needs it — the same arrangement the overlay
 // key has, and for the same reason: the mesh can compose a service's configuration without ever
 // holding the key that configuration points at.
+//
+// **PKCS#8 PEM**, because that is the only reason the file exists. A key stored in this host's own
+// encoding is a key nothing can serve with: the mesh delivers a PEM certificate beside it, and
+// every TLS server there is — a web server's `ssl_certificate_key`, Go's `LoadX509KeyPair`,
+// `openssl s_server -key` — reads PEM and nothing else. It was base64 once, and the certificate
+// arrived, and the file was there, and nothing could start.
 func ServingKeyPath(statePath string) string {
 	return dirOf(statePath) + "/serving.key"
 }
@@ -78,21 +86,47 @@ func LoadServingKey(path string) (ServingKey, error) {
 		}
 		return ServingKey{}, err
 	}
-	private, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw)))
-	if err != nil || len(private) != ed25519.PrivateKeySize {
+	block, _ := pem.Decode(raw)
+	if block == nil {
+		// Distinguished from a corrupt key, because the remedy is different and the difference is
+		// invisible otherwise. A key this host wrote before it stored PEM is intact and unusable:
+		// nothing serving TLS can read it, and the machine fails at the moment something connects.
+		if _, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw))); err == nil {
+			return ServingKey{}, fmt.Errorf(
+				"%s holds a serving key in this host's old encoding, which nothing serving TLS "+
+					"can read. It is replaced by enrolling again, which generates one and tells "+
+					"the mesh about it", path)
+		}
 		return ServingKey{}, fmt.Errorf("%s is not a serving key", path)
 	}
-	key := ed25519.PrivateKey(private)
+	parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return ServingKey{}, fmt.Errorf("%s is not a serving key: %w", path, err)
+	}
+	key, isEd25519 := parsed.(ed25519.PrivateKey)
+	if !isEd25519 {
+		return ServingKey{}, fmt.Errorf(
+			"%s holds a %T, and a node serves with an Ed25519 key", path, parsed)
+	}
 	return ServingKey{
 		Public:  base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey)),
-		Private: base64.StdEncoding.EncodeToString(private),
+		Private: base64.StdEncoding.EncodeToString(key),
 	}, nil
 }
 
-// WriteServingKey puts the private half where configuration can point at it.
+// WriteServingKey puts the private half where configuration can point at it, as PKCS#8 PEM.
 func WriteServingKey(path string, key ServingKey) error {
 	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
 		return err
 	}
-	return os.WriteFile(path, []byte(key.Private+"\n"), 0o600)
+	raw, err := base64.StdEncoding.DecodeString(key.Private)
+	if err != nil || len(raw) != ed25519.PrivateKeySize {
+		return fmt.Errorf("this is not a serving key to write")
+	}
+	encoded, err := x509.MarshalPKCS8PrivateKey(ed25519.PrivateKey(raw))
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path,
+		pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encoded}), 0o600)
 }

internal/identity/serving_test.go

@@ -0,0 +1,71 @@
+package identity
+
+import (
+	"crypto/ed25519"
+	"crypto/x509"
+	"encoding/pem"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// The whole reason the file exists is that something else reads it.
+//
+// A key in this host's own encoding is intact, unusable, and indistinguishable from a working one
+// until the moment a client connects — the mesh delivers the certificate, the file is there with
+// the right permissions, and the server will not start.
+func TestTheServingKeyIsWrittenInTheFormatAServerReads(t *testing.T) {
+	made, err := GenerateServingKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(t.TempDir(), "serving.key")
+	if err := WriteServingKey(path, made); err != nil {
+		t.Fatal(err)
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	block, _ := pem.Decode(raw)
+	if block == nil {
+		t.Fatalf("the serving key is not PEM, so nothing serving TLS can read it:\n%s", raw)
+	}
+	parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		t.Fatalf("the serving key is PEM and not a key: %v", err)
+	}
+	// And it is the key that was written, not merely a key — a file that round-trips through the
+	// wrong half would certify a public key the machine cannot prove it holds.
+	if _, isEd25519 := parsed.(ed25519.PrivateKey); !isEd25519 {
+		t.Fatalf("the serving key is a %T", parsed)
+	}
+	read, err := LoadServingKey(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if read.Public != made.Public {
+		t.Fatal("the key read back is not the key written, so the mesh would certify the wrong one")
+	}
+}
+
+// The old encoding is refused by name, because the remedy is different from a corrupt file and
+// the difference is invisible from the outside.
+func TestAServingKeyInTheOldEncodingIsNamedRatherThanCalledCorrupt(t *testing.T) {
+	made, err := GenerateServingKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(t.TempDir(), "serving.key")
+	if err := os.WriteFile(path, []byte(made.Private+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err = LoadServingKey(path)
+	if err == nil {
+		t.Fatal("a key nothing can serve with was accepted")
+	}
+	if !strings.Contains(err.Error(), "enrolling again") {
+		t.Fatalf("refused without naming the remedy: %v", err)
+	}
+}