From 5fad1f89cf625380415d55104193eb74d19dde39 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 25 Sep 2026 20:37:03 +0200 Subject: [PATCH] route-proxy: a second authority for internal names, and a target a route names the scheme of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal aliases were served over plain HTTP only — correctly refused a public certificate (no public CA can validate a private name), and then left with nothing. The mesh has two authorities for its two name spaces (08-connectivity §2), so the proxy now takes an optional internal ACME directory and dispatches at the handshake by the same question HostPolicy already answers: which authority may certify this name at all. A route may also say its target speaks https, with insecure for a backend whose own certificate nothing would trust — the shape Mailu's webmail front needs, and the exception: everything else the mesh hands this proxy stays plain http on the private network. --- examples/route-proxy/main.go | 163 ++++++++++++++++++++++------ examples/route-proxy/routes_test.go | 87 +++++++++++++++ 2 files changed, 219 insertions(+), 31 deletions(-) diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go index 78f52f7..c8f16a3 100644 --- a/examples/route-proxy/main.go +++ b/examples/route-proxy/main.go @@ -104,6 +104,19 @@ func onlyWhatTheMeshSaid(held *table) autocert.HostPolicy { } } +// onlyInternalNamesTheMeshSaid is onlyWhatTheMeshSaid's mirror for the internal authority — the +// same quota-spending concern applies even to an authority with no rate limit of its own, because +// an order for a name this proxy does not actually route is a bug worth refusing rather than +// serving. +func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy { + return func(_ context.Context, host string) error { + if held.eligibleForInternalACME(host) { + return nil + } + return fmt.Errorf("no internal-only route for %q in this mesh, so no certificate is asked for", host) + } +} + // what the mesh writes: the contributions file, one entry per consumer. type given struct { Given []contribution `json:"given"` @@ -149,6 +162,11 @@ type rule struct { policy policy to *httputil.ReverseProxy target string + // insecure skips certificate verification when target is reached over https. For a backend + // that terminates TLS with its own certificate this proxy has no reason to trust — Mailu's + // webmail front is the first of these — never for anything reached over plain http, where + // there is nothing to verify in the first place. + insecure bool } // table is what the proxy is currently serving, replaced whole whenever the file changes. @@ -187,6 +205,9 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) { continue } r.to = httputil.NewSingleHostReverseProxy(where) + if r.insecure { + r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + } kept = append(kept, r) } if len(kept) == 0 { @@ -256,6 +277,21 @@ func (t *table) routed(host string) bool { return len(t.to[bareHost(host)]) > 0 } +// eligibleForInternalACME says whether this proxy may ask its *internal* authority for a +// certificate for this name — every host it routes that is not also a route's public `name`. +// +// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity +// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is +// composed only from `to` and `public`, which routesFrom already builds correctly — a host never +// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be +// tracked to tell the two apart. +func (t *table) eligibleForInternalACME(host string) bool { + t.mu.RLock() + defer t.mu.RUnlock() + bare := bareHost(host) + return len(t.to[bare]) > 0 && !t.public[bare] +} + // bareHost is the name without the port, lower-cased. // // The port is not part of the name: a request to app.example:8080 is for app.example. Lower-cased @@ -333,16 +369,81 @@ func run() error { return fmt.Errorf("TLS_LISTEN is set and ACME_CACHE is not: certificates need somewhere " + "to persist, or every restart orders them again") } - client := &acme.Client{DirectoryURL: issuer()} - // An issuer that is not one of the public ones serves its own API over TLS with a certificate - // nothing trusts yet — the lab's, or an internal step-ca. Trusting it is a deliberate act and - // names a file, rather than the client being told to skip verification: *skip* would also - // apply on the day this points at a public issuer, and nothing would say so. + publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")), + onlyWhatTheMeshSaid(held)) + if err != nil { + return err + } + log.Printf("issuing public certificates from %s, for whatever the mesh routes here", issuer()) + + // The internal authority is optional: unset means this proxy serves internal-only aliases over + // plain HTTP exactly as it always has, which is the standalone-binary default and a safe one — + // it asks nothing of an authority it was not told about. + var internalManager *autocert.Manager + if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" { + internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")), + onlyInternalNamesTheMeshSaid(held)) + if err != nil { + return fmt.Errorf("internal certificate authority: %w", err) + } + log.Printf("issuing internal certificates from %s, for every internal-only alias this routes", + directory) + } + + // Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge + // must be answered *at the name being certified*, which is why issuance happens on the node + // that is publicly reachable rather than wherever the workload runs. Each manager's own + // HTTPHandler answers only the tokens it is itself expecting and falls through otherwise — for + // a token neither authority recognises, plain routing takes over, which is what lets a + // consumer's own ACME client — Mailu's, certifying its own name for a protocol this proxy never + // proxies — go on answering its own challenge through an ordinary path-scoped route. + port80 := publicManager.HTTPHandler(handler(held)) + if internalManager != nil { + port80 = publicManager.HTTPHandler(internalManager.HTTPHandler(handler(held))) + } + go func() { + if err := http.ListenAndServe(listen, port80); err != nil { + log.Printf("plain HTTP stopped: %v", err) + } + }() + + tlsConfig := publicManager.TLSConfig() + if internalManager != nil { + // Dispatched by which authority may certify this name at all — the same question + // eligibleForInternalACME already answers, asked once more at handshake time rather than + // only when an order is placed, since a cached certificate is served here on every request + // and never goes through HostPolicy again. + fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate + tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if held.eligibleForInternalACME(hello.ServerName) { + return fromInternal(hello) + } + return fromPublic(hello) + } + } + + server := &http.Server{ + Addr: secure, + Handler: handler(held), + TLSConfig: tlsConfig, + } + return server.ListenAndServeTLS("", "") +} + +// newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and +// which names it may be asked to certify. +// +// **Trusting an authority names a file rather than skipping verification.** An issuer that is not +// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS +// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public +// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead. +func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) { + client := &acme.Client{DirectoryURL: directory} var root []byte - if bundle := strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")); bundle != "" { + if bundle != "" { read, err := os.ReadFile(bundle) if err != nil { - return fmt.Errorf("ACME_CA_BUNDLE names %s and it cannot be read: %w", bundle, err) + return nil, fmt.Errorf("the CA bundle names %s and it cannot be read: %w", bundle, err) } root = read // An empty bundle means the issuer's root is already in the system trust store — a public @@ -354,7 +455,7 @@ func run() error { if strings.TrimSpace(string(root)) != "" { pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(root) { - return fmt.Errorf("%s holds no certificate this can trust", bundle) + return nil, fmt.Errorf("%s holds no certificate this can trust", bundle) } client.HTTPClient = &http.Client{ Timeout: 30 * time.Second, @@ -363,31 +464,16 @@ func run() error { } } // Where this authority's account and certificates are kept. Per authority, not per proxy — see - // forThisAuthority, which is what makes a re-initialised CA heal itself. - mine := forThisAuthority(cache, issuer(), root) - manager := &autocert.Manager{ + // forThisAuthority, which is what makes a re-initialised CA heal itself, and what lets the + // public and internal authorities share one ACME_CACHE without colliding: they hash to + // different names because their directories differ. + mine := forThisAuthority(cache, directory, root) + return &autocert.Manager{ Cache: autocert.DirCache(mine), Prompt: autocert.AcceptTOS, - HostPolicy: onlyWhatTheMeshSaid(held), + HostPolicy: policy, Client: client, - } - log.Printf("issuing from %s into %s, for whatever the mesh routes here", issuer(), mine) - - // Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge - // must be answered *at the name being certified*, which is why issuance happens on the node - // that is publicly reachable rather than wherever the workload runs. - go func() { - if err := http.ListenAndServe(listen, manager.HTTPHandler(handler(held))); err != nil { - log.Printf("plain HTTP stopped: %v", err) - } - }() - - server := &http.Server{ - Addr: secure, - Handler: handler(held), - TLSConfig: manager.TLSConfig(), - } - return server.ListenAndServeTLS("", "") + }, nil } // forThisAuthority is where one ACME authority's account and certificates are kept. @@ -595,7 +681,22 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { if at == "" { at = "127.0.0.1" } - made.target = fmt.Sprintf("http://%s:%d", at, port) + // http unless the contribution says otherwise. A backend that terminates its own TLS + // with a certificate this proxy has no reason to trust — Mailu's webmail front is the + // first of these — is the reason `insecure` exists, and it stays the exception: every + // other target the mesh hands this proxy is a plain workload on the private network. + scheme, _ := c.Values["scheme"].(string) + scheme = strings.ToLower(strings.TrimSpace(scheme)) + if scheme == "" { + scheme = "http" + } + if scheme != "http" && scheme != "https" { + log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+ + "nor https; skipped", c.From, c.Node, name, scheme) + continue + } + made.insecure, _ = c.Values["insecure"].(bool) + made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port) } out[host] = append(out[host], made) diff --git a/examples/route-proxy/routes_test.go b/examples/route-proxy/routes_test.go index b4d4bfa..36d49a5 100644 --- a/examples/route-proxy/routes_test.go +++ b/examples/route-proxy/routes_test.go @@ -130,6 +130,68 @@ func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) { } } +// A route may name a target reached over https, for a backend that terminates its own TLS — the +// shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise. +func TestARouteMayTargetHttps(t *testing.T) { + routes, _, err := routesFrom(write(t, `{"given":[ + {"from":"mail","node":"anchor","at":"anchor.internal", + "values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if targetOf(routes, "mail.example") != "https://anchor.internal:7443" { + t.Fatalf("an https target was not built as one: %v", routes) + } + if !routes["mail.example"][0].insecure { + t.Fatal("insecure was declared and not carried onto the rule") + } +} + +// A scheme that is neither http nor https is refused rather than guessed at. +func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) { + routes, _, err := routesFrom(write(t, `{"given":[ + {"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if len(routes) != 0 { + t.Fatalf("a route with an unusable scheme was served: %v", routes) + } +} + +// End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still +// reached when the route declared `insecure`, and the response comes back through unmodified. +func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) { + workload := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("the workload, over its own TLS")) + })) + defer workload.Close() + target := strings.TrimPrefix(workload.URL, "https://") + + held := newTable() + routes := map[string][]rule{"mail.example": {{target: "https://" + target, insecure: true}}} + held.set(routes, allPublic(routes)) + + proxy := httptest.NewServer(handler(held)) + defer proxy.Close() + + asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil) + if err != nil { + t.Fatal(err) + } + asked.Host = "mail.example" + answer, err := http.DefaultClient.Do(asked) + if err != nil { + t.Fatal(err) + } + defer answer.Body.Close() + if answer.StatusCode != http.StatusOK { + t.Fatalf("an insecure https backend was not reached: %d", answer.StatusCode) + } +} + // End to end through the proxy itself: a request for the name reaches the workload, and a name // nobody asked for is refused in a way that says what IS served. func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) { @@ -271,6 +333,31 @@ func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) { } } +// A certificate is asked of the *internal* authority only for a name that is routed here and is +// not a route's own public name — the internal-network alias, never the route it accompanies. +func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) { + routes, public, err := routesFrom(write(t, `{"given":[ + {"from":"app","node":"anchor","at":"anchor.internal", + "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} + ]}`)) + if err != nil { + t.Fatal(err) + } + held := newTable() + held.set(routes, public) + policy := onlyInternalNamesTheMeshSaid(held) + + if err := policy(context.Background(), "app.anchor.internal"); err != nil { + t.Errorf("the internal alias was refused by its own authority: %v", err) + } + if err := policy(context.Background(), "app.example"); err == nil { + t.Error("the internal authority certified a route's public name, which the public authority already covers") + } + if err := policy(context.Background(), "unrouted.internal"); err == nil { + t.Error("the internal authority certified a name nobody routed here") + } +} + // A route withdrawn stops being certifiable, without the proxy restarting. func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) { held := newTable() -- 2.54.0