mesh-controller/examples/route-proxy/main.go

// A reverse proxy, in the form the mesh expects one.
//
// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable
// declares it requires `route` and contributes the name it wants; the proxy provides `route` and
// receives every consumer that asked. It is the mirror of a database grant: there the consumer
// supplies a name and receives credentials, here it supplies a target and receives a name.
//
// **It is an example, not part of the control plane.** The control plane decides and never touches
// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this
// program. What lives here is that contract, written as something that runs so it can be read
// rather than described.
//
// **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used
// to say only where to send traffic, so this proxy applied nothing; the four things the ingress it
// replaces actually relies on are now part of the contribution. The set is closed at four, because
// an open middleware surface recreates the thing being replaced and is far harder to narrow later
// than a closed one is to widen.
//
// What it is given, written by the host from an ordinary declaration:
//
//	$ROUTES    every consumer, the name it asked for, and where the mesh says that machine is
//
// Each contribution's values carry the name and port as before, and optionally:
//
//	path       the path prefix this rule is scoped to; absent means every path
//	priority   which rule wins where two match; higher first, and the order is total
//	deny       refuse the request outright — the shape an incident mitigation needs
//	redirect   answer with a permanent redirect to this name, keeping the path and query
//	auth       the *path of a secret* holding `user:hash` lines, never the credential itself
//
// A host may appear more than once, which is what path scoping means: one rule refusing a path
// while another serves everything else on the same name.
//
// **`auth` names a secret and never holds one.** A declaration carrying a credential is refused
// outright rather than served unprotected, and a secret that cannot be read makes the route refuse
// rather than open — a gate that cannot check is not a gate that opens.
//
// It re-reads on change rather than being restarted, for the same reason the provisioner does:
// a route arriving or leaving is an ordinary event and must not drop the connections of every
// other workload.
package main

import (
	"bytes"
	"context"
	"crypto/sha256"
	"crypto/subtle"
	"crypto/tls"
	"crypto/x509"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/netip"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/crypto/acme"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/crypto/bcrypt"
)

// Where public certificates come from when nothing says otherwise.
//
// **Staging, deliberately** (novox/hq 04-ISSUES/004). Production issuance is rate-limited per
// domain and per account, the quota does not replenish quickly, and exhausting it removes the
// ability to issue a certificate somebody actually needs. Defaulting to production would leave
// the safe path depending on remembering to opt out of it, on exactly the work most likely to
// iterate — standing up a node, changing how names resolve.
//
// A staging certificate is trusted by no browser, so the mistake announces itself on the first
// request rather than a fortnight later at the rate limit.
const stagingDirectory = "https://acme-staging-v02.api.letsencrypt.org/directory"

// issuer is the ACME directory to ask. The lab points this at its own issuer; a node serving real
// traffic points it at production, and says so explicitly.
func issuer() string {
	if named := strings.TrimSpace(os.Getenv("ACME_DIRECTORY")); named != "" {
		return named
	}
	return stagingDirectory
}

// onlyWhatTheMeshSaid refuses to obtain a certificate for a name this proxy was not given.
//
// **The policy that stops a quota from being spent by accident.** Without it, anything that can
// reach port 443 and send a name triggers an issuance attempt for it — so a scan, or one
// misconfigured client, becomes a stream of failed orders against the account's rate limit. What
// this proxy may certify is exactly what the mesh told it to route, which is already the answer
// to what it may serve.
func onlyWhatTheMeshSaid(held *table) autocert.HostPolicy {
	return func(_ context.Context, host string) error {
		if held.eligibleForACME(host) {
			return nil
		}
		return fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", host)
	}
}

// onlyInternalNamesTheMeshSaid is onlyWhatTheMeshSaid's mirror for the internal authority — the
// same quota-spending concern applies even to an authority with no rate limit of its own, because
// an order for a name this proxy does not actually route is a bug worth refusing rather than
// serving.
func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy {
	return func(_ context.Context, host string) error {
		if held.eligibleForInternalACME(host) {
			return nil
		}
		return fmt.Errorf("no internal-only route for %q in this mesh, so no certificate is asked for", host)
	}
}

// what the mesh writes: the contributions file, one entry per consumer.
type given struct {
	Given []contribution `json:"given"`
}

type contribution struct {
	From string `json:"from"`
	Node string `json:"node"`
	// At is where that machine is on the private network. The mesh knows it; a proxy that had to
	// build it from the node name would be a naming convention copied into every provider.
	At     string         `json:"at"`
	Values map[string]any `json:"values"`
}

// policy is what a rule does with a request that matched it.
//
// **Decided by the mesh, not here** (novox/hq ADR 0108). A route grant used to hand back a name and
// say nothing about what the name admitted, so this proxy admitted everything. The set is closed at
// four — authentication, refusal, path scoping, redirect — because an open middleware surface
// recreates the thing being replaced and is far harder to narrow later than a closed one is to widen.
type policy struct {
	// deny refuses the request outright, whatever it is.
	deny bool
	// redirectTo answers with a permanent redirect instead of proxying. The request's own path and
	// query are carried across, which is what canonicalising one public name onto another means.
	redirectTo string
	// users is what a request must present, read at load time from the secret the declaration
	// *named*. A declaration never carries the credential itself.
	users map[string]string
	// sealed is set when authentication was declared and the secret could not be read. The rule then
	// refuses everything and says why.
	//
	// **Fail closed.** The alternative — serve the route unauthenticated because the gate is
	// missing — turns an unreadable file into a silently public admin surface, which is the exact
	// outcome ADR 0108 exists to prevent. A gate that cannot check is not a gate that opens.
	sealed string
}

// rule is one way a host may be routed. A host may have several, which is what path scoping means.
type rule struct {
	path     string // "" matches every path
	priority int
	policy   policy
	to       *httputil.ReverseProxy
	target   string
	// insecure skips certificate verification when target is reached over https. For a backend
	// that terminates TLS with its own certificate this proxy has no reason to trust — Mailu's
	// webmail front is the first of these — never for anything reached over plain http, where
	// there is nothing to verify in the first place.
	insecure bool
	// maxRequestBody is the largest body, in bytes, this route carries. Zero is no limit, which is
	// what every route gets by saying nothing: this proxy has never limited a body, and a default
	// arriving with the field would change every route that never asked for one.
	//
	// **Configuration, not a policy** (novox/hq ADR 0108 closed that set at four). It belongs beside
	// `insecure` for the same reason `insecure` is not a policy: both tune how this proxy carries a
	// request to a backend, rather than deciding what the name admits or who may reach it. A
	// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
	// a proxy's own default refuses them long before the workload is reached.
	maxRequestBody int64
}

// table is what the proxy is currently serving, replaced whole whenever the file changes.
//
// Replaced rather than merged: the file is the whole truth about who has a route, so merging
// would keep serving a name whose module was unassigned — which is the stale-route fault
// 08-connectivity lists as open, reintroduced one level down.
//
// Keyed by host to an *ordered* list rather than to one target, because two of the four policies
// need a single host routed more than one way: a refusal on a path the ordinary route also matches,
// and a certificate-challenge path on a host that otherwise serves a workload.
type table struct {
	mu sync.RWMutex
	to map[string][]rule
	// public is which routed hosts are eligible for a real certificate — every host reached as a
	// route's own `name`, never one reached only as its `internal-name`. A private alias can never
	// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
	// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
	public map[string]bool
	// inside is where a request must come from to be served a name that is only internal: every
	// machine's address on the private network, as the mesh issued it in this proxy's membership
	// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
	inside sources
}

// sources is who may be served an internal name: the private network's addresses as the mesh
// issued them. The machine itself is always inside — anything on a machine may call anything on it
// (novox/hq ADR 0144) — so loopback needs no entry.
type sources []netip.Prefix

// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does
// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to
// fewer machines than the mesh said, and say nothing.
func sourcesOf(mesh []string) (sources, error) {
	var out sources
	for _, entry := range mesh {
		entry = strings.TrimSpace(entry)
		if prefix, err := netip.ParsePrefix(entry); err == nil {
			out = append(out, prefix.Masked())
			continue
		}
		addr, err := netip.ParseAddr(entry)
		if err != nil {
			return nil, fmt.Errorf("%q is not an address on the private network", entry)
		}
		addr = addr.Unmap()
		out = append(out, netip.PrefixFrom(addr, addr.BitLen()))
	}
	return out, nil
}

// holds says whether a request from this remote address came from the mesh or the machine itself.
//
// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source
// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request
// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a
// mesh address leave by the tunnel, never back to the claimant.
func (s sources) holds(remote string) bool {
	host := remote
	if h, _, err := net.SplitHostPort(remote); err == nil {
		host = h
	}
	addr, err := netip.ParseAddr(host)
	if err != nil {
		return false
	}
	addr = addr.Unmap()
	if addr.IsLoopback() {
		return true
	}
	for _, prefix := range s {
		if prefix.Contains(addr) {
			return true
		}
	}
	return false
}

func (t *table) set(routes map[string][]rule, public map[string]bool) {
	made := map[string][]rule{}
	for host, rules := range routes {
		kept := make([]rule, 0, len(rules))
		for _, r := range rules {
			// A rule that only refuses or only redirects has nowhere to send anything, and needs
			// nowhere: it answers by itself.
			if r.policy.deny || r.policy.redirectTo != "" {
				kept = append(kept, r)
				continue
			}
			where, err := url.Parse(r.target)
			if err != nil {
				log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
				continue
			}
			r.to = httputil.NewSingleHostReverseProxy(where)
			if r.insecure {
				r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
			}
			kept = append(kept, r)
		}
		if len(kept) == 0 {
			continue
		}
		inOrder(kept)
		made[host] = kept
	}
	t.mu.Lock()
	t.to = made
	t.public = public
	t.mu.Unlock()
}

// inOrder puts the rules for one host into the order they are matched in, and does so totally.
//
// **Equal priorities must resolve identically every time** (ADR 0108). Sorting only by priority
// leaves rules that share one in whatever order the map produced, so the same declaration would
// serve differently between restarts — a proxy that is not reproducible. Longest path first within a
// priority is also the intuitive reading: the more specific rule wins. The last two keys exist only
// to make the order total.
func inOrder(rules []rule) {
	sort.SliceStable(rules, func(i, j int) bool {
		a, b := rules[i], rules[j]
		if a.priority != b.priority {
			return a.priority > b.priority
		}
		if len(a.path) != len(b.path) {
			return len(a.path) > len(b.path)
		}
		if a.path != b.path {
			return a.path < b.path
		}
		return a.target < b.target
	})
}

// find is the rule that answers this request, or nothing if the host is not routed here at all.
func (t *table) find(host, path string) (rule, bool) {
	t.mu.RLock()
	defer t.mu.RUnlock()
	for _, r := range t.to[bareHost(host)] {
		if r.path == "" || strings.HasPrefix(path, r.path) {
			return r, true
		}
	}
	return rule{}, false
}

// routed says whether this proxy serves the name at all, whatever the path.
//
// Separate from find because certificate issuance is a question about the *name*: a host whose only
// rules are path-scoped is still a name this proxy answers to, and still needs a certificate.
// eligibleForACME says whether this proxy may ask a certificate authority for this name — every
// host reached as a route's own public `name`, never one reached only as its `internal-name`
// alias, which no public CA can ever validate.
func (t *table) eligibleForACME(host string) bool {
	t.mu.RLock()
	defer t.mu.RUnlock()
	bare := bareHost(host)
	return len(t.to[bare]) > 0 && t.public[bare]
}

func (t *table) routed(host string) bool {
	t.mu.RLock()
	defer t.mu.RUnlock()
	return len(t.to[bareHost(host)]) > 0
}

// eligibleForInternalACME says whether this proxy may ask its *internal* authority for a
// certificate for this name — every host it routes that is not also a route's public `name`.
//
// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity
// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is
// composed only from `to` and `public`, which routesFrom already builds correctly — a host never
// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be
// tracked to tell the two apart.
func (t *table) eligibleForInternalACME(host string) bool {
	t.mu.RLock()
	defer t.mu.RUnlock()
	bare := bareHost(host)
	return len(t.to[bare]) > 0 && !t.public[bare]
}

// bareHost is the name without the port, lower-cased.
//
// The port is not part of the name: a request to app.example:8080 is for app.example. Lower-cased
// because a Host header is not case-sensitive, and a route that only answers the spelling in the
// manifest answers half the requests made to it.
func bareHost(host string) string {
	if h, _, err := net.SplitHostPort(host); err == nil {
		host = h
	}
	return strings.ToLower(host)
}

// hiddenFrom says whether this host must look unrouted to a request from this address: it is
// only an internal name, and the request did not come from the private network.
//
// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true**
// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from
// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does.
// Answered exactly as a name that was never routed, so an outsider learns nothing from asking.
func (t *table) hiddenFrom(host, remote string) bool {
	if !t.eligibleForInternalACME(host) {
		return false
	}
	t.mu.RLock()
	defer t.mu.RUnlock()
	return !t.inside.holds(remote)
}

// setInside replaces who the mesh is, as the membership said.
func (t *table) setInside(inside sources) {
	t.mu.Lock()
	t.inside = inside
	t.mu.Unlock()
}

// namesSeenFrom is what this proxy says it serves to a request from this address — every routed
// name, less the internal-only ones when the request came from outside.
func (t *table) namesSeenFrom(remote string) []string {
	out := []string{}
	for _, name := range t.names() {
		if !t.hiddenFrom(name, remote) {
			out = append(out, name)
		}
	}
	return out
}

func (t *table) names() []string {
	t.mu.RLock()
	defer t.mu.RUnlock()
	out := make([]string, 0, len(t.to))
	for name := range t.to {
		out = append(out, name)
	}
	sort.Strings(out)
	return out
}

func main() {
	if err := run(); err != nil {
		fmt.Fprintln(os.Stderr, "mesh-route-proxy: "+err.Error())
		os.Exit(1)
	}
}

func run() error {
	path := strings.TrimSpace(os.Getenv("ROUTES"))
	if path == "" {
		return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing")
	}
	listen := strings.TrimSpace(os.Getenv("LISTEN"))
	if listen == "" {
		listen = ":80"
	}

	held := newTable()
	// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
	// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
	// it an internal name is served to this machine and to nobody else — refused, never opened.
	fromBus := &atomic.Bool{}
	if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" {
		go followMembership(credential, held, fromBus)
	} else {
		log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+
			"internal is served to this machine alone", path)
	}
	read := func() {
		if fromBus.Load() {
			return
		}
		routes, public, err := routesFrom(path)
		if err != nil {
			// Kept serving what it had. A file being rewritten is momentarily unreadable, and
			// dropping every route because one read landed mid-write would turn an ordinary
			// event into an outage.
			log.Printf("cannot read %s, keeping what is already served: %v", path, err)
			return
		}
		held.set(routes, public)
		log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", "))
	}
	read()

	go func() {
		for range time.Tick(2 * time.Second) {
			read()
		}
	}()

	// TLS is opt-in. A proxy with no `TLS_LISTEN` serves plain HTTP exactly as before — which is
	// what an internal-only mesh wants, and what the mesh's own certificate authority already
	// covers for names inside it (novox/hq 08-connectivity). This is for names reachable from
	// outside, where the authority has to be one the world already trusts.
	secure := strings.TrimSpace(os.Getenv("TLS_LISTEN"))
	if secure == "" {
		return http.ListenAndServe(listen, handler(held))
	}

	cache := strings.TrimSpace(os.Getenv("ACME_CACHE"))
	if cache == "" {
		// Refused rather than defaulted. Without somewhere durable to keep them, every restart
		// orders new certificates — which works, silently, until the rate limit says it does not.
		return fmt.Errorf("TLS_LISTEN is set and ACME_CACHE is not: certificates need somewhere " +
			"to persist, or every restart orders them again")
	}
	publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
		onlyWhatTheMeshSaid(held))
	if err != nil {
		return err
	}
	log.Printf("issuing public certificates from %s, for whatever the mesh routes here", issuer())

	// The internal authority is optional: unset means this proxy serves internal-only aliases over
	// plain HTTP exactly as it always has, which is the standalone-binary default and a safe one —
	// it asks nothing of an authority it was not told about.
	var internalManager *autocert.Manager
	if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
		internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
			onlyInternalNamesTheMeshSaid(held))
		if err != nil {
			return fmt.Errorf("internal certificate authority: %w", err)
		}
		log.Printf("issuing internal certificates from %s, for every internal-only alias this routes",
			directory)
	}

	// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
	// must be answered *at the name being certified*, which is why issuance happens on the node
	// that is publicly reachable rather than wherever the workload runs.
	//
	// **autocert's own HTTPHandler does not fall through on the challenge path.** For a token it
	// does not hold it answers 404 itself; its fallback only ever sees non-challenge paths — which
	// is exactly the predecessor's fault, the edge owning `/.well-known/acme-challenge` outright,
	// rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute
	// probes each manager and hands a token neither authority recognises to plain routing, which
	// is what lets a consumer's own ACME client — Mailu's, certifying its own name for a protocol
	// this proxy never proxies — answer its own challenge through an ordinary path-scoped route.
	port80 := tokenOrRoute(handler(held), publicManager)
	if internalManager != nil {
		port80 = tokenOrRoute(handler(held), publicManager, internalManager)
	}
	go func() {
		if err := http.ListenAndServe(listen, port80); err != nil {
			log.Printf("plain HTTP stopped: %v", err)
		}
	}()

	tlsConfig := publicManager.TLSConfig()
	tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager)

	server := &http.Server{
		Addr:      secure,
		Handler:   handler(held),
		TLSConfig: tlsConfig,
	}
	return server.ListenAndServeTLS("", "")
}

// tokenOrRoute serves port 80: each manager answers the challenge tokens it is itself holding,
// and a token none of them holds is routed like any other request instead of being 404'd at the
// edge.
//
// autocert gives no way to ask "is this your token?" — its HTTPHandler both answers and refuses —
// so each manager is probed against a buffered writer and its refusal (404 on the challenge path)
// is discarded in favour of the next candidate. The probe is cheap: the handler answers from
// memory, and the path only carries traffic while an issuance is actually running.
func tokenOrRoute(routes http.Handler, managers ...*autocert.Manager) http.Handler {
	const challengePrefix = "/.well-known/acme-challenge/"
	// Non-challenge paths never reach a manager at all; autocert's tryHTTP01 switch still has to
	// be armed, which HTTPHandler is the only exported way to do.
	probes := make([]http.Handler, len(managers))
	for i, m := range managers {
		probes[i] = m.HTTPHandler(routes)
	}
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if !strings.HasPrefix(r.URL.Path, challengePrefix) {
			routes.ServeHTTP(w, r)
			return
		}
		for _, probe := range probes {
			buffered := &probedResponse{header: make(http.Header)}
			probe.ServeHTTP(buffered, r)
			// Two shapes of "not mine": 404, a token this manager is not holding — and 403, a
			// name its host policy would never certify at all (autocert checks the policy before
			// the token, so the internal authority answers 403 for every public name).
			if buffered.status == http.StatusNotFound || buffered.status == http.StatusForbidden {
				continue
			}
			buffered.replayTo(w)
			return
		}
		routes.ServeHTTP(w, r) // no authority holds it: the workload behind a routed path may
	})
}

// probedResponse buffers one handler's answer so a refusal can be discarded unseen.
type probedResponse struct {
	header http.Header
	status int
	body   bytes.Buffer
}

func (p *probedResponse) Header() http.Header { return p.header }

func (p *probedResponse) WriteHeader(status int) {
	if p.status == 0 {
		p.status = status
	}
}

func (p *probedResponse) Write(b []byte) (int, error) {
	if p.status == 0 {
		p.status = http.StatusOK
	}
	return p.body.Write(b)
}

func (p *probedResponse) replayTo(w http.ResponseWriter) {
	for k, vs := range p.header {
		for _, v := range vs {
			w.Header().Add(k, v)
		}
	}
	status := p.status
	if status == 0 {
		status = http.StatusOK
	}
	w.WriteHeader(status)
	_, _ = w.Write(p.body.Bytes())
}

// newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and
// which names it may be asked to certify.
//
// **Trusting an authority names a file rather than skipping verification.** An issuer that is not
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
	client := &acme.Client{DirectoryURL: directory}
	var root []byte
	if bundle != "" {
		read, err := os.ReadFile(bundle)
		if err != nil {
			return nil, fmt.Errorf("the CA bundle names %s and it cannot be read: %w", bundle, err)
		}
		root = read
		// An empty bundle means the issuer's root is already in the system trust store — a public
		// authority whose root ships with the OS, pointed at by a provider that serves an empty
		// root (novox/hq ADR 0066). The mesh always writes the bundle file, so it exists and holds
		// nothing; that is the signal to fall back to the system roots, the same as if nothing had
		// named a bundle at all. A file that holds bytes but no certificate is still a
		// misconfiguration and is refused, because there the operator meant to trust something.
		if strings.TrimSpace(string(root)) != "" {
			pool := x509.NewCertPool()
			if !pool.AppendCertsFromPEM(root) {
				return nil, fmt.Errorf("%s holds no certificate this can trust", bundle)
			}
			client.HTTPClient = &http.Client{
				Timeout:   30 * time.Second,
				Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}},
			}
		}
	}
	// Where this authority's account and certificates are kept. Per authority, not per proxy — see
	// forThisAuthority, which is what makes a re-initialised CA heal itself, and what lets the
	// public and internal authorities share one ACME_CACHE without colliding: they hash to
	// different names because their directories differ.
	mine := forThisAuthority(cache, directory, root)
	return &autocert.Manager{
		Cache:      autocert.DirCache(mine),
		Prompt:     autocert.AcceptTOS,
		HostPolicy: policy,
		Client:     client,
	}, nil
}

// forThisAuthority is where one ACME authority's account and certificates are kept.
//
// **A cached ACME account belongs to the authority that issued it, and nothing in the cache says
// so** (novox/hq ADR 0066). autocert keeps its account key at one fixed name — `acme_account+key` —
// in whatever directory it is given, and reuses it for ever. That is right while the authority stays
// the same and silently wrong the moment it does not: an internal CA that is re-initialised is a new
// authority with a new root, it has never heard of the account in the cache, and every attempt to
// use it is rejected. autocert has no path back from that. Nothing is retried, nothing is
// re-registered, no order ever reaches the CA — issuance simply stops, with no error anybody sees,
// until a person deletes the directory by hand and finds out that was the answer.
//
// So the directory is named after the authority instead of being shared by all of them. The name is
// a digest of the two things that identify one: the directory URL, and the root this proxy was told
// to verify it with. Re-initialising the CA produces a new root; the mesh delivers it as a changed
// bundle; this proxy restarts on that file and lands in a directory with no account in it, so
// autocert registers afresh and orders again. **The healing is that the question "is this account
// still valid" never has to be asked** — an account is only ever found where it is still valid.
//
// It also fixes a latent one of the same shape: pointing ACME_DIRECTORY at production after testing
// against staging reused the staging account, because the cache had no idea they were different.
//
// The old directories stay on disk, unused. Left rather than deleted: they are the only copy of
// certificates that may still be valid, and this program is not the thing that should decide a
// certificate is finished with.
func forThisAuthority(cache, directory string, root []byte) string {
	sum := sha256.Sum256([]byte(directory + "\x00" + string(bytes.TrimSpace(root))))
	return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
}

// certificateFor picks the certificate a handshake is answered with.
//
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than only when
// an order is placed, since a cached certificate is served here on every request and never goes
// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the
// private network asking for a name that is only internal: the certificate would name it.
func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error),
	internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
	var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error)
	if internalManager != nil {
		fromInternal = internalManager.TLSConfig().GetCertificate
	}
	return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		if held.eligibleForInternalACME(hello.ServerName) {
			if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
				return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
					hello.ServerName)
			}
			if fromInternal != nil {
				return fromInternal(hello)
			}
		}
		return fromPublic(hello)
	}
}

// newTable is an empty routing table.
func newTable() *table {
	return &table{to: map[string][]rule{}}
}

// handler is the proxy itself, separated so it can be driven by a test without a listener.
func handler(held *table) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hidden := held.hiddenFrom(r.Host, r.RemoteAddr)
		matched, known := held.find(r.Host, r.URL.Path)
		if hidden || !known {
			// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
			// are different things, and a proxy that says only "not found" makes an operator go
			// and read the mesh to tell them apart. What it is serving is the answer to both.
			//
			// And since a host may now be routed only on some paths, those are a third thing:
			// saying "no route for this name" while listing that very name as served is a
			// contradiction an operator would have to disbelieve the proxy to get past.
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(http.StatusNotFound)
			if !hidden && held.routed(r.Host) {
				fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
					bareHost(r.Host), r.URL.Path)
				return
			}
			fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
				r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", "))
			return
		}

		switch {
		case matched.policy.sealed != "":
			// Declared a gate, cannot check it. Refused, and says why — an operator reading this
			// learns the secret is missing, rather than wondering why a protected name is 503.
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintf(w, "this route requires authentication and its credentials cannot be read: %s\n",
				matched.policy.sealed)
			return

		case matched.policy.deny:
			http.Error(w, "this path is not served to you", http.StatusForbidden)
			return

		case matched.policy.redirectTo != "":
			http.Redirect(w, r, canonical(matched.policy.redirectTo, r.URL), http.StatusMovedPermanently)
			return

		case len(matched.policy.users) > 0 && !allowed(matched.policy.users, r):
			// The realm is the name asked for, so a browser's prompt says which route it is for.
			w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q, charset=\"UTF-8\"", bareHost(r.Host)))
			http.Error(w, "unauthorized", http.StatusUnauthorized)
			return
		}

		if matched.maxRequestBody > 0 {
			// Refused on the declared length where there is one, so an upload that cannot succeed
			// is answered before it is carried; and capped while reading for a chunked body, which
			// declares no length at all. Without the second, a limit is advice.
			if r.ContentLength > matched.maxRequestBody {
				http.Error(w, fmt.Sprintf("request body too large for this route: %d bytes is the most it carries",
					matched.maxRequestBody), http.StatusRequestEntityTooLarge)
				return
			}
			r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
		}

		matched.to.ServeHTTP(w, r)
	})
}

// canonical is where a redirect sends this request.
//
// The declaration names the destination *name*; the request keeps its own path and query. That is
// what canonicalising one public name onto another means — a link to a page under the old name has
// to arrive at the same page under the new one, or the redirect silently loses every deep link.
func canonical(to string, from *url.URL) string {
	where, err := url.Parse(to)
	if err != nil {
		return to
	}
	if where.Path == "" || where.Path == "/" {
		where.Path = from.Path
	}
	if where.RawQuery == "" {
		where.RawQuery = from.RawQuery
	}
	return where.String()
}

// allowed says whether the request presented credentials this route accepts.
//
// **Every path costs one bcrypt comparison**, including an unknown user, which is why the miss
// compares against a fixed hash rather than returning early. Returning early would make an unknown
// user measurably faster than a known one with a wrong password, and that difference is a way to
// enumerate the users of a route from outside it.
func allowed(users map[string]string, r *http.Request) bool {
	// A hash of nothing anybody knows. Its only job is to cost what a real comparison costs.
	const absent = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"
	user, password, ok := r.BasicAuth()
	if !ok {
		return false
	}
	want, known := users[user]
	if !known {
		want = absent
	}
	if err := bcrypt.CompareHashAndPassword([]byte(want), []byte(password)); err != nil {
		return false
	}
	// `known` is checked after the comparison, not instead of it, so the timing is the same either
	// way. subtle.ConstantTimeByteEq keeps the branch from being the thing that differs.
	return subtle.ConstantTimeByteEq(boolByte(known), 1) == 1
}

func boolByte(b bool) byte {
	if b {
		return 1
	}
	return 0
}

// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
// only through `internal-name` never appears there.
//
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches
// decides which names the mesh composes, so an endpoint that reaches only the private network
// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is
// served under its internal name and certified by the internal authority. Only a route with
// neither name has nothing to be served under (novox/hq issue 191).
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, nil, err
	}
	var said given
	if err := json.Unmarshal(raw, &said); err != nil {
		return nil, nil, err
	}
	routes, public := routesOf(said.Given)
	return routes, public, nil
}

// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public
// names — the same whether the contributions came in the file or in the membership.
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
	out := map[string][]rule{}
	public := map[string]bool{}
	for _, c := range contributions {
		name, _ := c.Values["name"].(string)
		name = strings.TrimSpace(name)
		internal, _ := c.Values["internal-name"].(string)
		internal = strings.TrimSpace(internal)
		if name == "" && internal == "" {
			log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
			continue
		}
		// What the route is called in a log line: its public name when it has one.
		called := name
		if called == "" {
			called = internal
		}

		made := rule{path: asPath(c.Values["path"])}
		if p, ok := asWhole(c.Values["priority"]); ok {
			made.priority = p
		}
		made.policy.deny, _ = c.Values["deny"].(bool)
		made.policy.redirectTo, _ = c.Values["redirect"].(string)

		if named, carried := c.Values["auth"].(string); carried && strings.TrimSpace(named) != "" {
			// **A declaration names a secret; it never holds one** (ADR 0108). Refused rather than
			// tolerated, and the whole rule is dropped rather than served unprotected — the
			// rejected option cannot come back by accident, which is the failure this check exists
			// to make impossible.
			if looksLikeACredential(named) {
				log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
					"than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
					c.From, c.Node, called)
				continue
			}
			users, err := usersFrom(named)
			if err != nil {
				// Fail closed: the rule is kept so the name stays routed and answers, and it
				// answers by refusing. Dropping it instead would make the name 404 and read as a
				// withdrawn route rather than an unreadable secret.
				made.policy.sealed = err.Error()
			}
			made.policy.users = users
		}

		// Only a rule that actually proxies needs somewhere to send the request.
		if !made.policy.deny && made.policy.redirectTo == "" {
			port, ok := asPort(c.Values["port"])
			if !ok {
				log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
					c.From, c.Node, called)
				continue
			}
			// Where the mesh says that machine is. Empty means it is this one — a workload beside
			// the proxy is ordinary, and reaching it over loopback is both correct and the only
			// thing that works when there is no private network.
			at := c.At
			if at == "" {
				at = "127.0.0.1"
			}
			// http unless the contribution says otherwise. A backend that terminates its own TLS
			// with a certificate this proxy has no reason to trust — Mailu's webmail front is the
			// first of these — is the reason `insecure` exists, and it stays the exception: every
			// other target the mesh hands this proxy is a plain workload on the private network.
			scheme, _ := c.Values["scheme"].(string)
			scheme = strings.ToLower(strings.TrimSpace(scheme))
			if scheme == "" {
				scheme = "http"
			}
			if scheme != "http" && scheme != "https" {
				log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
					"nor https; skipped", c.From, c.Node, called, scheme)
				continue
			}
			made.insecure, _ = c.Values["insecure"].(bool)
			// A limit this proxy cannot read is a route it does not serve, named like a port that
			// is not a port. Serving it without the limit would carry exactly what the module said
			// not to carry, and report success doing it.
			if asked, said := c.Values["max-request-body"]; said {
				bytes, whole := asWhole(asked)
				if !whole || bytes <= 0 {
					log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
						"not a whole positive number of bytes; skipped", c.From, c.Node, called, asked)
					continue
				}
				made.maxRequestBody = int64(bytes)
			}
			made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
		}

		if name != "" {
			host := strings.ToLower(name)
			out[host] = append(out[host], made)
			public[host] = true
		}

		// The internal-network name, the same rule under a second host — a predecessor proxy
		// answered both for one route, as a convenience (reaching a service over the VPN without a
		// public TLS round trip), not as an access boundary; composing it here restores exactly
		// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
		// 0056's internalDomain half) — the same "nothing to join a label to" case the public name
		// already has. And the only name, when the endpoint reaches no further than the private
		// network.
		if internal != "" {
			out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
		}
	}
	return out, public
}

// asWhole is any whole number the mesh wrote, whatever its magnitude.
//
// **Not asPort.** Priority was read with the port reader first, which caps at 65535 — so a rule
// declared at a priority above that silently became priority 0 and stopped shadowing the route it
// exists to shadow. The one real rule this has to reproduce is declared at 100000, so the bug was
// exactly load-bearing. A priority is an ordering, not a port: it has no range.
func asWhole(v any) (int, bool) {
	switch n := v.(type) {
	case float64:
		// JSON makes a float of every number, so a non-integral one was not meant as a priority.
		if n != float64(int(n)) {
			return 0, false
		}
		return int(n), true
	case int:
		return n, true
	}
	return 0, false
}

// asPath is the path prefix a rule is scoped to, or "" for every path.
func asPath(v any) string {
	p, _ := v.(string)
	p = strings.TrimSpace(p)
	if p == "" {
		return ""
	}
	if !strings.HasPrefix(p, "/") {
		p = "/" + p
	}
	return p
}

// looksLikeACredential is the check that keeps a secret out of a declaration.
//
// It errs towards refusing: a value holding a `:` (the htpasswd separator) or opening with a bcrypt
// identifier is a credential, not a path, and no filesystem path the mesh writes needs either. A
// false refusal is a loud log and a route that does not serve; a false accept is a credential
// committed to a declaration, which is the thing being prevented.
func looksLikeACredential(v string) bool {
	v = strings.TrimSpace(v)
	return strings.Contains(v, ":") || strings.HasPrefix(v, "$2")
}

// usersFrom reads the credentials the mesh mounted, in the one format every htpasswd already is.
func usersFrom(path string) (map[string]string, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("cannot read the secret named for this route: %w", err)
	}
	users := map[string]string{}
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		user, hash, ok := strings.Cut(line, ":")
		if !ok || user == "" || hash == "" {
			continue
		}
		users[user] = hash
	}
	if len(users) == 0 {
		return nil, fmt.Errorf("the secret named for this route holds no usable credentials")
	}
	return users, nil
}

// asPort accepts what JSON makes of a number, which is a float even when it was written 8080.
func asPort(v any) (int, bool) {
	switch n := v.(type) {
	case float64:
		if n < 1 || n > 65535 {
			return 0, false
		}
		return int(n), true
	case int:
		if n < 1 || n > 65535 {
			return 0, false
		}
		return n, true
	}
	return 0, false
}