From ef618e819971dc9d211b9ba13c1a466783b8b24d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 09:36:21 +0100 Subject: [PATCH] nats: the bus's certificate command moves here from mesh-controller with the same output, made once and checked, so the bus module makes its own certificate when a mesh is raised from nothing (issue 774, design 47 L3 s11) --- .../cmd/mesh-nats-snapshot/certificate.go | 174 ++++++++++++++++++ .../mesh-nats-snapshot/certificate_test.go | 138 ++++++++++++++ modules/nats/cmd/mesh-nats-snapshot/main.go | 4 + 3 files changed, 316 insertions(+) create mode 100644 modules/nats/cmd/mesh-nats-snapshot/certificate.go create mode 100644 modules/nats/cmd/mesh-nats-snapshot/certificate_test.go diff --git a/modules/nats/cmd/mesh-nats-snapshot/certificate.go b/modules/nats/cmd/mesh-nats-snapshot/certificate.go new file mode 100644 index 00000000..7d03bc9e --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/certificate.go @@ -0,0 +1,174 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +// The bus's own certificate, made by the mesh rather than borrowed from an image. Moved here from mesh-controller's +// `broker certificate` with the same output (design 47's L3 s11): the command belongs to the bus module, and the +// controller's binary is retired after its lease is handed away (ADR 0547 §11). Nothing changes at the lease write: the +// certificate in /var/lib/mesh-broker-tls stays, and every pin stays valid. +// +// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq +// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's +// image, which worked while the broker was one that happened to carry it and stopped the day the +// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be +// raised. Substituting another image the bundle names does not help — none of them carry it +// either. +// +// So the program that needs a certificate makes one. It is the mesh's own binary, already on the +// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the +// image it writes into but a mounted directory. +// +// **Self-signed, and that is the design** — a host pins this server's exact certificate and +// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at +// this moment in a bootstrap there is no mesh to ask one of. +// +// Idempotent, because the step is applied again on every reconcile and a second certificate would +// be one the hosts that pinned the first no longer believe. + +// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the +// loopback address the machine's own foundation dials. +var busCertificateNames = []string{"mesh-broker"} + +const busCertificateLife = 10 * 365 * 24 * time.Hour + +// busCertificate makes the bus's certificate in a directory, or says whether one is there. +// +// mesh-nats-snapshot certificate --into /tls make it if it is not there +// mesh-nats-snapshot certificate --check --into /tls exit non-zero unless a usable pair is +func busCertificate(args []string) error { + into, check := "", false + for i := 0; i < len(args); i++ { + switch args[i] { + case "--check": + check = true + case "--into": + if i+1 >= len(args) { + return errors.New("--into needs a directory") + } + into = args[i+1] + i++ + default: + return fmt.Errorf("certificate [--check] --into : %q", args[i]) + } + } + if into == "" { + return errors.New("certificate [--check] --into ") + } + crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key") + + if usable, err := busCertificateUsable(crt, key); err != nil { + return err + } else if usable { + fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt) + return nil + } + if check { + // Said as a failure, because that is what the caller asked: a bootstrap's verify runs + // this and a false answer is what makes the step run. + return fmt.Errorf("no usable certificate and key at %s", into) + } + return writeBusCertificate(crt, key) +} + +// busCertificateUsable says whether a certificate and its key are both there and parse. +// +// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted +// between the two files leaves behind, and a step that treated it as done would hand the server a +// certificate with no key and report success. +func busCertificateUsable(crt, key string) (bool, error) { + certPEM, err := os.ReadFile(crt) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + keyPEM, err := os.ReadFile(key) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if _, err := tlsPairParses(certPEM, keyPEM); err != nil { + return false, nil + } + return true, nil +} + +func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(certPEM) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("not a certificate") + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, errors.New("not a key") + } + if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil { + if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil { + return nil, err + } + } + return certificate, nil +} + +func writeBusCertificate(crt, key string) error { + private, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return err + } + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: busCertificateNames[0]}, + DNSNames: busCertificateNames, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(busCertificateLife), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private) + if err != nil { + return err + } + pkcs8, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + return err + } + + // **The key first, and only then the certificate**, so the pair a reader finds is never a + // certificate whose key has not been written yet — the one order in which an interruption + // leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable). + if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil { + return err + } + if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil { + return err + } + fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n", + busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key) + return nil +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go b/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go new file mode 100644 index 00000000..e03df592 --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "os" + "path/filepath" + "strings" + "testing" +) + +// L3 s11 (design 47): the bus's certificate command, moved from mesh-controller with the same output, made once, and +// its check false without a usable pair. Its cases, as the controller's tests had them, are rows of one test. +func TestTheBusCertificate(t *testing.T) { + for _, row := range []struct { + name string + run func(*testing.T) + }{ + {"it loads as a server loads it", loadsAsAServersWould}, + {"it is made once", isMadeOnce}, + {"the check is false until there is a pair", checkIsFalseUntilAPair}, + {"a certificate without its key is not usable", withoutItsKeyIsNotUsable}, + {"where to write is required", whereToWriteIsRequired}, + } { + t.Run(row.name, row.run) + } +} + +// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for +// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is +// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once. + +func loadsAsAServersWould(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatalf("the bus could not be given a certificate: %v", err) + } + + // The check a cheaper test would not make. The key was present and valid and the server could + // not start, once, because nothing loaded the pair the way a server loads it + // (novox/hq 04-ISSUES/014). + pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")) + if err != nil { + t.Fatalf("a TLS server cannot load what was written: %v", err) + } + leaf := pair.Leaf + if leaf == nil { + if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil { + t.Fatal(err) + } + } + if err := leaf.VerifyHostname("mesh-broker"); err != nil { + t.Errorf("the certificate is not for the name the bus is reached by: %v", err) + } + if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" { + t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses) + } + + // The key is not readable by anything else on the machine; the certificate is public and is. + key, err := os.Stat(filepath.Join(into, "tls.key")) + if err != nil { + t.Fatal(err) + } + if key.Mode().Perm() != 0o600 { + t.Errorf("the key is %v", key.Mode().Perm()) + } + crt, err := os.Stat(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if crt.Mode().Perm() != 0o644 { + t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm()) + } +} + +// **Made once.** The step is applied again on every reconcile, and a second certificate is one the +// hosts that pinned the first no longer believe (novox/hq ADR 0004). +func isMadeOnce(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + again, err := os.ReadFile(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if string(first) != string(again) { + t.Fatal("running it twice replaced the certificate every host had pinned") + } +} + +// The verify half: false before, true after, which is what makes the bootstrap run the step at all. +func checkIsFalseUntilAPair(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--check", "--into", into}); err == nil { + t.Fatal("an empty directory reported a usable certificate") + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--check", "--into", into}); err != nil { + t.Fatalf("the certificate it just made does not satisfy its own check: %v", err) + } +} + +// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that +// called it done would hand the server a certificate with no key and report success. +func withoutItsKeyIsNotUsable(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(into, "tls.key")); err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--check", "--into", into}); err == nil { + t.Fatal("a certificate with no key passed the check") + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil { + t.Fatalf("it did not replace the unusable pair: %v", err) + } +} + +func whereToWriteIsRequired(t *testing.T) { + if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") { + t.Fatalf("it did not ask where to write: %v", err) + } +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/main.go b/modules/nats/cmd/mesh-nats-snapshot/main.go index f71e49d8..a3232c77 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/main.go +++ b/modules/nats/cmd/mesh-nats-snapshot/main.go @@ -17,6 +17,7 @@ // mesh-nats-snapshot verify [--from FILE] // mesh-nats-snapshot restore --into DIR [--account MESH] [--from FILE] // mesh-nats-snapshot restore --server URL --credential FILE [--from FILE] +// mesh-nats-snapshot certificate [--check] --into DIR // // **Read-only on the live bus.** A snapshot neither pauses nor reconfigures a stream; the server // keeps accepting writes while it streams the copy out. The credential it uses is the bus's own @@ -47,6 +48,8 @@ func main() { err = runVerify(os.Args[2:]) case "restore": err = runRestore(os.Args[2:]) + case "certificate": + err = busCertificate(os.Args[2:]) case "-h", "--help", "help": usage() return @@ -67,6 +70,7 @@ func usage() { verify [--from bus.tar] every archive against the manifest restore --into DIR [--account MESH] [--from bus.tar] into a new store directory, by a server of its own restore --server URL --credential FILE [--from bus.tar] into a running server that holds none of them + certificate [--check] --into DIR the bus's self-signed certificate, made once The credential is the JSON the mesh seals to the bus's own module: url, fingerprint, user, password. `)