route-proxy: the challenge path falls through for real

autocert's HTTPHandler answers 404 itself for a token it does not hold
and never consults its fallback on the challenge path — the
predecessor's exact fault, rediscovered live when Mailu's renewal died
behind this proxy on cutover day. tokenOrRoute probes each authority
against a buffered writer and hands a token none of them holds to plain
routing, so a consumer's own ACME client answers its own challenge
through an ordinary path-scoped route. Four tests pin it, including the
cache-key shape a restart-surviving token actually has.

examples/route-proxy/challenge_test.go

@@ -0,0 +1,86 @@
+package main
+
+// The challenge path falls through for real. autocert's own HTTPHandler answers 404 itself for a
+// token it does not hold and never consults its fallback on the challenge path — the
+// predecessor's fault, the edge owning /.well-known/acme-challenge outright, rediscovered live
+// when Mailu's renewal died behind this proxy on cutover day (2026-09-26). These tests pin the
+// three behaviours tokenOrRoute exists for.
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"golang.org/x/crypto/acme/autocert"
+)
+
+func routedTo(t *testing.T, marker string) http.Handler {
+	t.Helper()
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		if _, err := w.Write([]byte(marker)); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func TestATokenNoAuthorityHoldsIsRoutedNot404d(t *testing.T) {
+	m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
+	h := tokenOrRoute(routedTo(t, "the workload answered"), m)
+
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/somebody-elses-token", nil))
+
+	if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
+		t.Fatalf("a token no authority holds must reach plain routing; got %d %q", rec.Code, rec.Body.String())
+	}
+}
+
+func TestATokenAManagerHoldsIsAnsweredByIt(t *testing.T) {
+	// autocert reads a token it does not have in memory from its cache, under "<token>+http-01" —
+	// which is also how a token would survive the manager restarting mid-issuance.
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "held-token+http-01"), []byte("the-key-authorization"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
+	h := tokenOrRoute(routedTo(t, "must not be reached"), m)
+
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/held-token", nil))
+
+	if rec.Code != http.StatusOK || rec.Body.String() != "the-key-authorization" {
+		t.Fatalf("the manager holding a token answers it; got %d %q", rec.Code, rec.Body.String())
+	}
+}
+
+func TestASecondAuthorityIsProbedBeforeRouting(t *testing.T) {
+	first := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "internal-token+http-01"), []byte("internal-key"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	second := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
+	h := tokenOrRoute(routedTo(t, "must not be reached"), first, second)
+
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://git.internal/.well-known/acme-challenge/internal-token", nil))
+
+	if rec.Code != http.StatusOK || rec.Body.String() != "internal-key" {
+		t.Fatalf("the second authority's token is found by probing past the first; got %d %q", rec.Code, rec.Body.String())
+	}
+}
+
+func TestAnOrdinaryPathNeverTouchesTheChallengeMachinery(t *testing.T) {
+	m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
+	h := tokenOrRoute(routedTo(t, "routed"), m)
+
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://site.example/index.html", nil))
+
+	if rec.Code != http.StatusOK || rec.Body.String() != "routed" {
+		t.Fatalf("an ordinary path goes straight to routing; got %d %q", rec.Code, rec.Body.String())
+	}
+}

examples/route-proxy/main.go

@@ -392,14 +392,18 @@ func run() error {
 
 	// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
 	// must be answered *at the name being certified*, which is why issuance happens on the node
-	// that is publicly reachable rather than wherever the workload runs. Each manager's own
+	// that is publicly reachable rather than wherever the workload runs.
-	// HTTPHandler answers only the tokens it is itself expecting and falls through otherwise — for
+	//
-	// a token neither authority recognises, plain routing takes over, which is what lets a
+	// **autocert's own HTTPHandler does not fall through on the challenge path.** For a token it
-	// consumer's own ACME client — Mailu's, certifying its own name for a protocol this proxy never
+	// does not hold it answers 404 itself; its fallback only ever sees non-challenge paths — which
-	// proxies — go on answering its own challenge through an ordinary path-scoped route.
+	// is exactly the predecessor's fault, the edge owning `/.well-known/acme-challenge` outright,
-	port80 := publicManager.HTTPHandler(handler(held))
+	// rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute
+	// probes each manager and hands a token neither authority recognises to plain routing, which
+	// is what lets a consumer's own ACME client — Mailu's, certifying its own name for a protocol
+	// this proxy never proxies — answer its own challenge through an ordinary path-scoped route.
+	port80 := tokenOrRoute(handler(held), publicManager)
 	if internalManager != nil {
-		port80 = publicManager.HTTPHandler(internalManager.HTTPHandler(handler(held)))
+		port80 = tokenOrRoute(handler(held), publicManager, internalManager)
 	}
 	go func() {
 		if err := http.ListenAndServe(listen, port80); err != nil {
@@ -430,6 +434,76 @@ func run() error {
 	return server.ListenAndServeTLS("", "")
 }
 
+// tokenOrRoute serves port 80: each manager answers the challenge tokens it is itself holding,
+// and a token none of them holds is routed like any other request instead of being 404'd at the
+// edge.
+//
+// autocert gives no way to ask "is this your token?" — its HTTPHandler both answers and refuses —
+// so each manager is probed against a buffered writer and its refusal (404 on the challenge path)
+// is discarded in favour of the next candidate. The probe is cheap: the handler answers from
+// memory, and the path only carries traffic while an issuance is actually running.
+func tokenOrRoute(routes http.Handler, managers ...*autocert.Manager) http.Handler {
+	const challengePrefix = "/.well-known/acme-challenge/"
+	// Non-challenge paths never reach a manager at all; autocert's tryHTTP01 switch still has to
+	// be armed, which HTTPHandler is the only exported way to do.
+	probes := make([]http.Handler, len(managers))
+	for i, m := range managers {
+		probes[i] = m.HTTPHandler(routes)
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, challengePrefix) {
+			routes.ServeHTTP(w, r)
+			return
+		}
+		for _, probe := range probes {
+			buffered := &probedResponse{header: make(http.Header)}
+			probe.ServeHTTP(buffered, r)
+			if buffered.status == http.StatusNotFound {
+				continue // not this manager's token
+			}
+			buffered.replayTo(w)
+			return
+		}
+		routes.ServeHTTP(w, r) // no authority holds it: the workload behind a routed path may
+	})
+}
+
+// probedResponse buffers one handler's answer so a refusal can be discarded unseen.
+type probedResponse struct {
+	header http.Header
+	status int
+	body   bytes.Buffer
+}
+
+func (p *probedResponse) Header() http.Header { return p.header }
+
+func (p *probedResponse) WriteHeader(status int) {
+	if p.status == 0 {
+		p.status = status
+	}
+}
+
+func (p *probedResponse) Write(b []byte) (int, error) {
+	if p.status == 0 {
+		p.status = http.StatusOK
+	}
+	return p.body.Write(b)
+}
+
+func (p *probedResponse) replayTo(w http.ResponseWriter) {
+	for k, vs := range p.header {
+		for _, v := range vs {
+			w.Header().Add(k, v)
+		}
+	}
+	status := p.status
+	if status == 0 {
+		status = http.StatusOK
+	}
+	w.WriteHeader(status)
+	_, _ = w.Write(p.body.Bytes())
+}
+
 // newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and
 // which names it may be asked to certify.
 //