diff --git a/examples/route-proxy/authority_test.go b/examples/route-proxy/authority_test.go new file mode 100644 index 0000000..0f70a27 --- /dev/null +++ b/examples/route-proxy/authority_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +// The lab's first root, and the one a re-initialised CA generates in its place. +const ( + firstRoot = "-----BEGIN CERTIFICATE-----\nMIIBeFIRST\n-----END CERTIFICATE-----\n" + secondRoot = "-----BEGIN CERTIFICATE-----\nMIIBeSECOND\n-----END CERTIFICATE-----\n" +) + +// A re-initialised CA does not need somebody to delete the cache by hand. +// +// autocert keeps its account key at one fixed name and reuses it for ever. When an internal CA is +// re-initialised it has never heard of that account, rejects every use of it, and autocert has no +// path back: nothing is re-registered, no order reaches the CA, and issuance stops with nothing +// saying why. Naming the cache after the authority means the account is only ever found where it is +// still valid — the new root lands in a directory with no account in it, and autocert registers. +func TestANewCARootMeansANewAccountCache(t *testing.T) { + const directory = "https://anchor.internal/acme/acme/directory" + before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot)) + after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot)) + if before == after { + t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before) + } +} + +// And the SAME authority keeps the account it registered, restart after restart. +// +// This is the whole reason ACME_CACHE is required in the first place: an account and its +// certificates that did not persist would be re-ordered on every restart, which works silently until +// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the +// mesh writes that file, and a trailing newline coming or going must not throw away an account. +func TestTheSameAuthorityKeepsItsAccount(t *testing.T) { + const directory = "https://anchor.internal/acme/acme/directory" + first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot)) + again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n")) + if first != again { + t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s", + first, again) + } +} + +// Staging and production are different authorities, and were sharing one account. +// +// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against +// staging reused the staging account, because the cache had no idea they were different. +func TestStagingAndProductionDoNotShareAnAccount(t *testing.T) { + staging := forThisAuthority("/acme", stagingDirectory, nil) + production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil) + if staging == production { + t.Errorf("two issuers share one account: %s", staging) + } +} + +// It stays inside the directory the mesh gave it, and is a plain name. +// +// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or +// that carried a separator out of the PEM — would put an account somewhere nothing persists. +func TestTheAccountCacheStaysWhereTheMeshPutIt(t *testing.T) { + const cache = "/var/lib/route-proxy/acme" + got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot)) + if !strings.HasPrefix(got, cache+"/") { + t.Fatalf("the account cache is not under %s: %s", cache, got) + } + name := strings.TrimPrefix(got, cache+"/") + if name != filepath.Base(got) || strings.ContainsAny(name, "/.") { + t.Errorf("the account cache is not a plain name: %q", name) + } +} diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go index 3235d18..397d65f 100644 --- a/examples/route-proxy/main.go +++ b/examples/route-proxy/main.go @@ -20,9 +20,12 @@ package main import ( + "bytes" "context" + "crypto/sha256" "crypto/tls" "crypto/x509" + "encoding/hex" "encoding/json" "fmt" "log" @@ -31,6 +34,7 @@ import ( "net/http/httputil" "net/url" "os" + "path/filepath" "sort" "strings" "sync" @@ -198,20 +202,22 @@ func run() error { // nothing trusts yet — the lab's, or an internal step-ca. Trusting it is a deliberate act and // names a file, rather than the client being told to skip verification: *skip* would also // apply on the day this points at a public issuer, and nothing would say so. + var root []byte if bundle := strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")); bundle != "" { - pem, err := os.ReadFile(bundle) + read, err := os.ReadFile(bundle) if err != nil { return fmt.Errorf("ACME_CA_BUNDLE names %s and it cannot be read: %w", bundle, err) } + root = read // An empty bundle means the issuer's root is already in the system trust store — a public // authority whose root ships with the OS, pointed at by a provider that serves an empty // root (novox/hq ADR 0056). The mesh always writes the bundle file, so it exists and holds // nothing; that is the signal to fall back to the system roots, the same as if nothing had // named a bundle at all. A file that holds bytes but no certificate is still a // misconfiguration and is refused, because there the operator meant to trust something. - if strings.TrimSpace(string(pem)) != "" { + if strings.TrimSpace(string(root)) != "" { pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(pem) { + if !pool.AppendCertsFromPEM(root) { return fmt.Errorf("%s holds no certificate this can trust", bundle) } client.HTTPClient = &http.Client{ @@ -220,13 +226,16 @@ func run() error { } } } + // Where this authority's account and certificates are kept. Per authority, not per proxy — see + // forThisAuthority, which is what makes a re-initialised CA heal itself. + mine := forThisAuthority(cache, issuer(), root) manager := &autocert.Manager{ - Cache: autocert.DirCache(cache), + Cache: autocert.DirCache(mine), Prompt: autocert.AcceptTOS, HostPolicy: onlyWhatTheMeshSaid(held), Client: client, } - log.Printf("issuing from %s, for whatever the mesh routes here", issuer()) + log.Printf("issuing from %s into %s, for whatever the mesh routes here", issuer(), mine) // Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge // must be answered *at the name being certified*, which is why issuance happens on the node @@ -245,6 +254,35 @@ func run() error { return server.ListenAndServeTLS("", "") } +// forThisAuthority is where one ACME authority's account and certificates are kept. +// +// **A cached ACME account belongs to the authority that issued it, and nothing in the cache says +// so** (novox/hq ADR 0056). autocert keeps its account key at one fixed name — `acme_account+key` — +// in whatever directory it is given, and reuses it for ever. That is right while the authority stays +// the same and silently wrong the moment it does not: an internal CA that is re-initialised is a new +// authority with a new root, it has never heard of the account in the cache, and every attempt to +// use it is rejected. autocert has no path back from that. Nothing is retried, nothing is +// re-registered, no order ever reaches the CA — issuance simply stops, with no error anybody sees, +// until a person deletes the directory by hand and finds out that was the answer. +// +// So the directory is named after the authority instead of being shared by all of them. The name is +// a digest of the two things that identify one: the directory URL, and the root this proxy was told +// to verify it with. Re-initialising the CA produces a new root; the mesh delivers it as a changed +// bundle; this proxy restarts on that file and lands in a directory with no account in it, so +// autocert registers afresh and orders again. **The healing is that the question "is this account +// still valid" never has to be asked** — an account is only ever found where it is still valid. +// +// It also fixes a latent one of the same shape: pointing ACME_DIRECTORY at production after testing +// against staging reused the staging account, because the cache had no idea they were different. +// +// The old directories stay on disk, unused. Left rather than deleted: they are the only copy of +// certificates that may still be valid, and this program is not the thing that should decide a +// certificate is finished with. +func forThisAuthority(cache, directory string, root []byte) string { + sum := sha256.Sum256([]byte(directory + "\x00" + string(bytes.TrimSpace(root)))) + return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) +} + // newTable is an empty routing table. func newTable() *table { return &table{to: map[string]*httputil.ReverseProxy{}, targets: map[string]string{}}