autocert's HTTPHandler answers 404 itself for a token it does not hold and never consults its fallback on the challenge path — the predecessor's exact fault, rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute probes each authority against a buffered writer and hands a token none of them holds to plain routing, so a consumer's own ACME client answers its own challenge through an ordinary path-scoped route. Four tests pin it, including the cache-key shape a restart-surviving token actually has.
875 lines
35 KiB
Go
875 lines
35 KiB
Go
// A reverse proxy, in the form the mesh expects one.
|
|
//
|
|
// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable
|
|
// declares it requires `route` and contributes the name it wants; the proxy provides `route` and
|
|
// receives every consumer that asked. It is the mirror of a database grant: there the consumer
|
|
// supplies a name and receives credentials, here it supplies a target and receives a name.
|
|
//
|
|
// **It is an example, not part of the control plane.** The control plane decides and never touches
|
|
// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this
|
|
// program. What lives here is that contract, written as something that runs so it can be read
|
|
// rather than described.
|
|
//
|
|
// **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used
|
|
// to say only where to send traffic, so this proxy applied nothing; the four things the ingress it
|
|
// replaces actually relies on are now part of the contribution. The set is closed at four, because
|
|
// an open middleware surface recreates the thing being replaced and is far harder to narrow later
|
|
// than a closed one is to widen.
|
|
//
|
|
// What it is given, written by the host from an ordinary declaration:
|
|
//
|
|
// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is
|
|
//
|
|
// Each contribution's values carry the name and port as before, and optionally:
|
|
//
|
|
// path the path prefix this rule is scoped to; absent means every path
|
|
// priority which rule wins where two match; higher first, and the order is total
|
|
// deny refuse the request outright — the shape an incident mitigation needs
|
|
// redirect answer with a permanent redirect to this name, keeping the path and query
|
|
// auth the *path of a secret* holding `user:hash` lines, never the credential itself
|
|
//
|
|
// A host may appear more than once, which is what path scoping means: one rule refusing a path
|
|
// while another serves everything else on the same name.
|
|
//
|
|
// **`auth` names a secret and never holds one.** A declaration carrying a credential is refused
|
|
// outright rather than served unprotected, and a secret that cannot be read makes the route refuse
|
|
// rather than open — a gate that cannot check is not a gate that opens.
|
|
//
|
|
// It re-reads on change rather than being restarted, for the same reason the provisioner does:
|
|
// a route arriving or leaving is an ordinary event and must not drop the connections of every
|
|
// other workload.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/subtle"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httputil"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/acme"
|
|
"golang.org/x/crypto/acme/autocert"
|
|
"golang.org/x/crypto/bcrypt"
|
|
)
|
|
|
|
// Where public certificates come from when nothing says otherwise.
|
|
//
|
|
// **Staging, deliberately** (novox/hq 04-ISSUES/004). Production issuance is rate-limited per
|
|
// domain and per account, the quota does not replenish quickly, and exhausting it removes the
|
|
// ability to issue a certificate somebody actually needs. Defaulting to production would leave
|
|
// the safe path depending on remembering to opt out of it, on exactly the work most likely to
|
|
// iterate — standing up a node, changing how names resolve.
|
|
//
|
|
// A staging certificate is trusted by no browser, so the mistake announces itself on the first
|
|
// request rather than a fortnight later at the rate limit.
|
|
const stagingDirectory = "https://acme-staging-v02.api.letsencrypt.org/directory"
|
|
|
|
// issuer is the ACME directory to ask. The lab points this at its own issuer; a node serving real
|
|
// traffic points it at production, and says so explicitly.
|
|
func issuer() string {
|
|
if named := strings.TrimSpace(os.Getenv("ACME_DIRECTORY")); named != "" {
|
|
return named
|
|
}
|
|
return stagingDirectory
|
|
}
|
|
|
|
// onlyWhatTheMeshSaid refuses to obtain a certificate for a name this proxy was not given.
|
|
//
|
|
// **The policy that stops a quota from being spent by accident.** Without it, anything that can
|
|
// reach port 443 and send a name triggers an issuance attempt for it — so a scan, or one
|
|
// misconfigured client, becomes a stream of failed orders against the account's rate limit. What
|
|
// this proxy may certify is exactly what the mesh told it to route, which is already the answer
|
|
// to what it may serve.
|
|
func onlyWhatTheMeshSaid(held *table) autocert.HostPolicy {
|
|
return func(_ context.Context, host string) error {
|
|
if held.eligibleForACME(host) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", host)
|
|
}
|
|
}
|
|
|
|
// onlyInternalNamesTheMeshSaid is onlyWhatTheMeshSaid's mirror for the internal authority — the
|
|
// same quota-spending concern applies even to an authority with no rate limit of its own, because
|
|
// an order for a name this proxy does not actually route is a bug worth refusing rather than
|
|
// serving.
|
|
func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy {
|
|
return func(_ context.Context, host string) error {
|
|
if held.eligibleForInternalACME(host) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("no internal-only route for %q in this mesh, so no certificate is asked for", host)
|
|
}
|
|
}
|
|
|
|
// what the mesh writes: the contributions file, one entry per consumer.
|
|
type given struct {
|
|
Given []contribution `json:"given"`
|
|
}
|
|
|
|
type contribution struct {
|
|
From string `json:"from"`
|
|
Node string `json:"node"`
|
|
// At is where that machine is on the private network. The mesh knows it; a proxy that had to
|
|
// build it from the node name would be a naming convention copied into every provider.
|
|
At string `json:"at"`
|
|
Values map[string]any `json:"values"`
|
|
}
|
|
|
|
// policy is what a rule does with a request that matched it.
|
|
//
|
|
// **Decided by the mesh, not here** (novox/hq ADR 0108). A route grant used to hand back a name and
|
|
// say nothing about what the name admitted, so this proxy admitted everything. The set is closed at
|
|
// four — authentication, refusal, path scoping, redirect — because an open middleware surface
|
|
// recreates the thing being replaced and is far harder to narrow later than a closed one is to widen.
|
|
type policy struct {
|
|
// deny refuses the request outright, whatever it is.
|
|
deny bool
|
|
// redirectTo answers with a permanent redirect instead of proxying. The request's own path and
|
|
// query are carried across, which is what canonicalising one public name onto another means.
|
|
redirectTo string
|
|
// users is what a request must present, read at load time from the secret the declaration
|
|
// *named*. A declaration never carries the credential itself.
|
|
users map[string]string
|
|
// sealed is set when authentication was declared and the secret could not be read. The rule then
|
|
// refuses everything and says why.
|
|
//
|
|
// **Fail closed.** The alternative — serve the route unauthenticated because the gate is
|
|
// missing — turns an unreadable file into a silently public admin surface, which is the exact
|
|
// outcome ADR 0108 exists to prevent. A gate that cannot check is not a gate that opens.
|
|
sealed string
|
|
}
|
|
|
|
// rule is one way a host may be routed. A host may have several, which is what path scoping means.
|
|
type rule struct {
|
|
path string // "" matches every path
|
|
priority int
|
|
policy policy
|
|
to *httputil.ReverseProxy
|
|
target string
|
|
// insecure skips certificate verification when target is reached over https. For a backend
|
|
// that terminates TLS with its own certificate this proxy has no reason to trust — Mailu's
|
|
// webmail front is the first of these — never for anything reached over plain http, where
|
|
// there is nothing to verify in the first place.
|
|
insecure bool
|
|
}
|
|
|
|
// table is what the proxy is currently serving, replaced whole whenever the file changes.
|
|
//
|
|
// Replaced rather than merged: the file is the whole truth about who has a route, so merging
|
|
// would keep serving a name whose module was unassigned — which is the stale-route fault
|
|
// 08-connectivity lists as open, reintroduced one level down.
|
|
//
|
|
// Keyed by host to an *ordered* list rather than to one target, because two of the four policies
|
|
// need a single host routed more than one way: a refusal on a path the ordinary route also matches,
|
|
// and a certificate-challenge path on a host that otherwise serves a workload.
|
|
type table struct {
|
|
mu sync.RWMutex
|
|
to map[string][]rule
|
|
// public is which routed hosts are eligible for a real certificate — every host reached as a
|
|
// route's own `name`, never one reached only as its `internal-name`. A private alias can never
|
|
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
|
|
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
|
|
public map[string]bool
|
|
}
|
|
|
|
func (t *table) set(routes map[string][]rule, public map[string]bool) {
|
|
made := map[string][]rule{}
|
|
for host, rules := range routes {
|
|
kept := make([]rule, 0, len(rules))
|
|
for _, r := range rules {
|
|
// A rule that only refuses or only redirects has nowhere to send anything, and needs
|
|
// nowhere: it answers by itself.
|
|
if r.policy.deny || r.policy.redirectTo != "" {
|
|
kept = append(kept, r)
|
|
continue
|
|
}
|
|
where, err := url.Parse(r.target)
|
|
if err != nil {
|
|
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
|
|
continue
|
|
}
|
|
r.to = httputil.NewSingleHostReverseProxy(where)
|
|
if r.insecure {
|
|
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
|
}
|
|
kept = append(kept, r)
|
|
}
|
|
if len(kept) == 0 {
|
|
continue
|
|
}
|
|
inOrder(kept)
|
|
made[host] = kept
|
|
}
|
|
t.mu.Lock()
|
|
t.to = made
|
|
t.public = public
|
|
t.mu.Unlock()
|
|
}
|
|
|
|
// inOrder puts the rules for one host into the order they are matched in, and does so totally.
|
|
//
|
|
// **Equal priorities must resolve identically every time** (ADR 0108). Sorting only by priority
|
|
// leaves rules that share one in whatever order the map produced, so the same declaration would
|
|
// serve differently between restarts — a proxy that is not reproducible. Longest path first within a
|
|
// priority is also the intuitive reading: the more specific rule wins. The last two keys exist only
|
|
// to make the order total.
|
|
func inOrder(rules []rule) {
|
|
sort.SliceStable(rules, func(i, j int) bool {
|
|
a, b := rules[i], rules[j]
|
|
if a.priority != b.priority {
|
|
return a.priority > b.priority
|
|
}
|
|
if len(a.path) != len(b.path) {
|
|
return len(a.path) > len(b.path)
|
|
}
|
|
if a.path != b.path {
|
|
return a.path < b.path
|
|
}
|
|
return a.target < b.target
|
|
})
|
|
}
|
|
|
|
// find is the rule that answers this request, or nothing if the host is not routed here at all.
|
|
func (t *table) find(host, path string) (rule, bool) {
|
|
t.mu.RLock()
|
|
defer t.mu.RUnlock()
|
|
for _, r := range t.to[bareHost(host)] {
|
|
if r.path == "" || strings.HasPrefix(path, r.path) {
|
|
return r, true
|
|
}
|
|
}
|
|
return rule{}, false
|
|
}
|
|
|
|
// routed says whether this proxy serves the name at all, whatever the path.
|
|
//
|
|
// Separate from find because certificate issuance is a question about the *name*: a host whose only
|
|
// rules are path-scoped is still a name this proxy answers to, and still needs a certificate.
|
|
// eligibleForACME says whether this proxy may ask a certificate authority for this name — every
|
|
// host reached as a route's own public `name`, never one reached only as its `internal-name`
|
|
// alias, which no public CA can ever validate.
|
|
func (t *table) eligibleForACME(host string) bool {
|
|
t.mu.RLock()
|
|
defer t.mu.RUnlock()
|
|
bare := bareHost(host)
|
|
return len(t.to[bare]) > 0 && t.public[bare]
|
|
}
|
|
|
|
func (t *table) routed(host string) bool {
|
|
t.mu.RLock()
|
|
defer t.mu.RUnlock()
|
|
return len(t.to[bareHost(host)]) > 0
|
|
}
|
|
|
|
// eligibleForInternalACME says whether this proxy may ask its *internal* authority for a
|
|
// certificate for this name — every host it routes that is not also a route's public `name`.
|
|
//
|
|
// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity
|
|
// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is
|
|
// composed only from `to` and `public`, which routesFrom already builds correctly — a host never
|
|
// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be
|
|
// tracked to tell the two apart.
|
|
func (t *table) eligibleForInternalACME(host string) bool {
|
|
t.mu.RLock()
|
|
defer t.mu.RUnlock()
|
|
bare := bareHost(host)
|
|
return len(t.to[bare]) > 0 && !t.public[bare]
|
|
}
|
|
|
|
// bareHost is the name without the port, lower-cased.
|
|
//
|
|
// The port is not part of the name: a request to app.example:8080 is for app.example. Lower-cased
|
|
// because a Host header is not case-sensitive, and a route that only answers the spelling in the
|
|
// manifest answers half the requests made to it.
|
|
func bareHost(host string) string {
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
return strings.ToLower(host)
|
|
}
|
|
|
|
func (t *table) names() []string {
|
|
t.mu.RLock()
|
|
defer t.mu.RUnlock()
|
|
out := make([]string, 0, len(t.to))
|
|
for name := range t.to {
|
|
out = append(out, name)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "mesh-route-proxy: "+err.Error())
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
path := strings.TrimSpace(os.Getenv("ROUTES"))
|
|
if path == "" {
|
|
return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing")
|
|
}
|
|
listen := strings.TrimSpace(os.Getenv("LISTEN"))
|
|
if listen == "" {
|
|
listen = ":80"
|
|
}
|
|
|
|
held := newTable()
|
|
read := func() {
|
|
routes, public, err := routesFrom(path)
|
|
if err != nil {
|
|
// Kept serving what it had. A file being rewritten is momentarily unreadable, and
|
|
// dropping every route because one read landed mid-write would turn an ordinary
|
|
// event into an outage.
|
|
log.Printf("cannot read %s, keeping what is already served: %v", path, err)
|
|
return
|
|
}
|
|
held.set(routes, public)
|
|
log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", "))
|
|
}
|
|
read()
|
|
|
|
go func() {
|
|
for range time.Tick(2 * time.Second) {
|
|
read()
|
|
}
|
|
}()
|
|
|
|
// TLS is opt-in. A proxy with no `TLS_LISTEN` serves plain HTTP exactly as before — which is
|
|
// what an internal-only mesh wants, and what the mesh's own certificate authority already
|
|
// covers for names inside it (novox/hq 08-connectivity). This is for names reachable from
|
|
// outside, where the authority has to be one the world already trusts.
|
|
secure := strings.TrimSpace(os.Getenv("TLS_LISTEN"))
|
|
if secure == "" {
|
|
return http.ListenAndServe(listen, handler(held))
|
|
}
|
|
|
|
cache := strings.TrimSpace(os.Getenv("ACME_CACHE"))
|
|
if cache == "" {
|
|
// Refused rather than defaulted. Without somewhere durable to keep them, every restart
|
|
// orders new certificates — which works, silently, until the rate limit says it does not.
|
|
return fmt.Errorf("TLS_LISTEN is set and ACME_CACHE is not: certificates need somewhere " +
|
|
"to persist, or every restart orders them again")
|
|
}
|
|
publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
|
|
onlyWhatTheMeshSaid(held))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Printf("issuing public certificates from %s, for whatever the mesh routes here", issuer())
|
|
|
|
// The internal authority is optional: unset means this proxy serves internal-only aliases over
|
|
// plain HTTP exactly as it always has, which is the standalone-binary default and a safe one —
|
|
// it asks nothing of an authority it was not told about.
|
|
var internalManager *autocert.Manager
|
|
if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
|
|
internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
|
|
onlyInternalNamesTheMeshSaid(held))
|
|
if err != nil {
|
|
return fmt.Errorf("internal certificate authority: %w", err)
|
|
}
|
|
log.Printf("issuing internal certificates from %s, for every internal-only alias this routes",
|
|
directory)
|
|
}
|
|
|
|
// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
|
|
// must be answered *at the name being certified*, which is why issuance happens on the node
|
|
// that is publicly reachable rather than wherever the workload runs.
|
|
//
|
|
// **autocert's own HTTPHandler does not fall through on the challenge path.** For a token it
|
|
// does not hold it answers 404 itself; its fallback only ever sees non-challenge paths — which
|
|
// is exactly the predecessor's fault, the edge owning `/.well-known/acme-challenge` outright,
|
|
// rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute
|
|
// probes each manager and hands a token neither authority recognises to plain routing, which
|
|
// is what lets a consumer's own ACME client — Mailu's, certifying its own name for a protocol
|
|
// this proxy never proxies — answer its own challenge through an ordinary path-scoped route.
|
|
port80 := tokenOrRoute(handler(held), publicManager)
|
|
if internalManager != nil {
|
|
port80 = tokenOrRoute(handler(held), publicManager, internalManager)
|
|
}
|
|
go func() {
|
|
if err := http.ListenAndServe(listen, port80); err != nil {
|
|
log.Printf("plain HTTP stopped: %v", err)
|
|
}
|
|
}()
|
|
|
|
tlsConfig := publicManager.TLSConfig()
|
|
if internalManager != nil {
|
|
// Dispatched by which authority may certify this name at all — the same question
|
|
// eligibleForInternalACME already answers, asked once more at handshake time rather than
|
|
// only when an order is placed, since a cached certificate is served here on every request
|
|
// and never goes through HostPolicy again.
|
|
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
|
|
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
|
if held.eligibleForInternalACME(hello.ServerName) {
|
|
return fromInternal(hello)
|
|
}
|
|
return fromPublic(hello)
|
|
}
|
|
}
|
|
|
|
server := &http.Server{
|
|
Addr: secure,
|
|
Handler: handler(held),
|
|
TLSConfig: tlsConfig,
|
|
}
|
|
return server.ListenAndServeTLS("", "")
|
|
}
|
|
|
|
// tokenOrRoute serves port 80: each manager answers the challenge tokens it is itself holding,
|
|
// and a token none of them holds is routed like any other request instead of being 404'd at the
|
|
// edge.
|
|
//
|
|
// autocert gives no way to ask "is this your token?" — its HTTPHandler both answers and refuses —
|
|
// so each manager is probed against a buffered writer and its refusal (404 on the challenge path)
|
|
// is discarded in favour of the next candidate. The probe is cheap: the handler answers from
|
|
// memory, and the path only carries traffic while an issuance is actually running.
|
|
func tokenOrRoute(routes http.Handler, managers ...*autocert.Manager) http.Handler {
|
|
const challengePrefix = "/.well-known/acme-challenge/"
|
|
// Non-challenge paths never reach a manager at all; autocert's tryHTTP01 switch still has to
|
|
// be armed, which HTTPHandler is the only exported way to do.
|
|
probes := make([]http.Handler, len(managers))
|
|
for i, m := range managers {
|
|
probes[i] = m.HTTPHandler(routes)
|
|
}
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if !strings.HasPrefix(r.URL.Path, challengePrefix) {
|
|
routes.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
for _, probe := range probes {
|
|
buffered := &probedResponse{header: make(http.Header)}
|
|
probe.ServeHTTP(buffered, r)
|
|
if buffered.status == http.StatusNotFound {
|
|
continue // not this manager's token
|
|
}
|
|
buffered.replayTo(w)
|
|
return
|
|
}
|
|
routes.ServeHTTP(w, r) // no authority holds it: the workload behind a routed path may
|
|
})
|
|
}
|
|
|
|
// probedResponse buffers one handler's answer so a refusal can be discarded unseen.
|
|
type probedResponse struct {
|
|
header http.Header
|
|
status int
|
|
body bytes.Buffer
|
|
}
|
|
|
|
func (p *probedResponse) Header() http.Header { return p.header }
|
|
|
|
func (p *probedResponse) WriteHeader(status int) {
|
|
if p.status == 0 {
|
|
p.status = status
|
|
}
|
|
}
|
|
|
|
func (p *probedResponse) Write(b []byte) (int, error) {
|
|
if p.status == 0 {
|
|
p.status = http.StatusOK
|
|
}
|
|
return p.body.Write(b)
|
|
}
|
|
|
|
func (p *probedResponse) replayTo(w http.ResponseWriter) {
|
|
for k, vs := range p.header {
|
|
for _, v := range vs {
|
|
w.Header().Add(k, v)
|
|
}
|
|
}
|
|
status := p.status
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
w.WriteHeader(status)
|
|
_, _ = w.Write(p.body.Bytes())
|
|
}
|
|
|
|
// newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and
|
|
// which names it may be asked to certify.
|
|
//
|
|
// **Trusting an authority names a file rather than skipping verification.** An issuer that is not
|
|
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
|
|
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
|
|
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
|
|
func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
|
|
client := &acme.Client{DirectoryURL: directory}
|
|
var root []byte
|
|
if bundle != "" {
|
|
read, err := os.ReadFile(bundle)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the CA bundle names %s and it cannot be read: %w", bundle, err)
|
|
}
|
|
root = read
|
|
// An empty bundle means the issuer's root is already in the system trust store — a public
|
|
// authority whose root ships with the OS, pointed at by a provider that serves an empty
|
|
// root (novox/hq ADR 0066). The mesh always writes the bundle file, so it exists and holds
|
|
// nothing; that is the signal to fall back to the system roots, the same as if nothing had
|
|
// named a bundle at all. A file that holds bytes but no certificate is still a
|
|
// misconfiguration and is refused, because there the operator meant to trust something.
|
|
if strings.TrimSpace(string(root)) != "" {
|
|
pool := x509.NewCertPool()
|
|
if !pool.AppendCertsFromPEM(root) {
|
|
return nil, fmt.Errorf("%s holds no certificate this can trust", bundle)
|
|
}
|
|
client.HTTPClient = &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}},
|
|
}
|
|
}
|
|
}
|
|
// Where this authority's account and certificates are kept. Per authority, not per proxy — see
|
|
// forThisAuthority, which is what makes a re-initialised CA heal itself, and what lets the
|
|
// public and internal authorities share one ACME_CACHE without colliding: they hash to
|
|
// different names because their directories differ.
|
|
mine := forThisAuthority(cache, directory, root)
|
|
return &autocert.Manager{
|
|
Cache: autocert.DirCache(mine),
|
|
Prompt: autocert.AcceptTOS,
|
|
HostPolicy: policy,
|
|
Client: client,
|
|
}, nil
|
|
}
|
|
|
|
// forThisAuthority is where one ACME authority's account and certificates are kept.
|
|
//
|
|
// **A cached ACME account belongs to the authority that issued it, and nothing in the cache says
|
|
// so** (novox/hq ADR 0066). autocert keeps its account key at one fixed name — `acme_account+key` —
|
|
// in whatever directory it is given, and reuses it for ever. That is right while the authority stays
|
|
// the same and silently wrong the moment it does not: an internal CA that is re-initialised is a new
|
|
// authority with a new root, it has never heard of the account in the cache, and every attempt to
|
|
// use it is rejected. autocert has no path back from that. Nothing is retried, nothing is
|
|
// re-registered, no order ever reaches the CA — issuance simply stops, with no error anybody sees,
|
|
// until a person deletes the directory by hand and finds out that was the answer.
|
|
//
|
|
// So the directory is named after the authority instead of being shared by all of them. The name is
|
|
// a digest of the two things that identify one: the directory URL, and the root this proxy was told
|
|
// to verify it with. Re-initialising the CA produces a new root; the mesh delivers it as a changed
|
|
// bundle; this proxy restarts on that file and lands in a directory with no account in it, so
|
|
// autocert registers afresh and orders again. **The healing is that the question "is this account
|
|
// still valid" never has to be asked** — an account is only ever found where it is still valid.
|
|
//
|
|
// It also fixes a latent one of the same shape: pointing ACME_DIRECTORY at production after testing
|
|
// against staging reused the staging account, because the cache had no idea they were different.
|
|
//
|
|
// The old directories stay on disk, unused. Left rather than deleted: they are the only copy of
|
|
// certificates that may still be valid, and this program is not the thing that should decide a
|
|
// certificate is finished with.
|
|
func forThisAuthority(cache, directory string, root []byte) string {
|
|
sum := sha256.Sum256([]byte(directory + "\x00" + string(bytes.TrimSpace(root))))
|
|
return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
|
|
}
|
|
|
|
// newTable is an empty routing table.
|
|
func newTable() *table {
|
|
return &table{to: map[string][]rule{}}
|
|
}
|
|
|
|
// handler is the proxy itself, separated so it can be driven by a test without a listener.
|
|
func handler(held *table) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
matched, known := held.find(r.Host, r.URL.Path)
|
|
if !known {
|
|
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
|
|
// are different things, and a proxy that says only "not found" makes an operator go
|
|
// and read the mesh to tell them apart. What it is serving is the answer to both.
|
|
//
|
|
// And since a host may now be routed only on some paths, those are a third thing:
|
|
// saying "no route for this name" while listing that very name as served is a
|
|
// contradiction an operator would have to disbelieve the proxy to get past.
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.WriteHeader(http.StatusNotFound)
|
|
if held.routed(r.Host) {
|
|
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
|
|
bareHost(r.Host), r.URL.Path)
|
|
return
|
|
}
|
|
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
|
|
r.Host, strings.Join(held.names(), ", "))
|
|
return
|
|
}
|
|
|
|
switch {
|
|
case matched.policy.sealed != "":
|
|
// Declared a gate, cannot check it. Refused, and says why — an operator reading this
|
|
// learns the secret is missing, rather than wondering why a protected name is 503.
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
fmt.Fprintf(w, "this route requires authentication and its credentials cannot be read: %s\n",
|
|
matched.policy.sealed)
|
|
return
|
|
|
|
case matched.policy.deny:
|
|
http.Error(w, "this path is not served to you", http.StatusForbidden)
|
|
return
|
|
|
|
case matched.policy.redirectTo != "":
|
|
http.Redirect(w, r, canonical(matched.policy.redirectTo, r.URL), http.StatusMovedPermanently)
|
|
return
|
|
|
|
case len(matched.policy.users) > 0 && !allowed(matched.policy.users, r):
|
|
// The realm is the name asked for, so a browser's prompt says which route it is for.
|
|
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q, charset=\"UTF-8\"", bareHost(r.Host)))
|
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
matched.to.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// canonical is where a redirect sends this request.
|
|
//
|
|
// The declaration names the destination *name*; the request keeps its own path and query. That is
|
|
// what canonicalising one public name onto another means — a link to a page under the old name has
|
|
// to arrive at the same page under the new one, or the redirect silently loses every deep link.
|
|
func canonical(to string, from *url.URL) string {
|
|
where, err := url.Parse(to)
|
|
if err != nil {
|
|
return to
|
|
}
|
|
if where.Path == "" || where.Path == "/" {
|
|
where.Path = from.Path
|
|
}
|
|
if where.RawQuery == "" {
|
|
where.RawQuery = from.RawQuery
|
|
}
|
|
return where.String()
|
|
}
|
|
|
|
// allowed says whether the request presented credentials this route accepts.
|
|
//
|
|
// **Every path costs one bcrypt comparison**, including an unknown user, which is why the miss
|
|
// compares against a fixed hash rather than returning early. Returning early would make an unknown
|
|
// user measurably faster than a known one with a wrong password, and that difference is a way to
|
|
// enumerate the users of a route from outside it.
|
|
func allowed(users map[string]string, r *http.Request) bool {
|
|
// A hash of nothing anybody knows. Its only job is to cost what a real comparison costs.
|
|
const absent = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"
|
|
user, password, ok := r.BasicAuth()
|
|
if !ok {
|
|
return false
|
|
}
|
|
want, known := users[user]
|
|
if !known {
|
|
want = absent
|
|
}
|
|
if err := bcrypt.CompareHashAndPassword([]byte(want), []byte(password)); err != nil {
|
|
return false
|
|
}
|
|
// `known` is checked after the comparison, not instead of it, so the timing is the same either
|
|
// way. subtle.ConstantTimeByteEq keeps the branch from being the thing that differs.
|
|
return subtle.ConstantTimeByteEq(boolByte(known), 1) == 1
|
|
}
|
|
|
|
func boolByte(b bool) byte {
|
|
if b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
|
|
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
|
|
// only through `internal-name` never appears there.
|
|
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
var said given
|
|
if err := json.Unmarshal(raw, &said); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
out := map[string][]rule{}
|
|
public := map[string]bool{}
|
|
for _, c := range said.Given {
|
|
name, _ := c.Values["name"].(string)
|
|
if name == "" {
|
|
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
|
|
continue
|
|
}
|
|
host := strings.ToLower(name)
|
|
public[host] = true
|
|
|
|
made := rule{path: asPath(c.Values["path"])}
|
|
if p, ok := asWhole(c.Values["priority"]); ok {
|
|
made.priority = p
|
|
}
|
|
made.policy.deny, _ = c.Values["deny"].(bool)
|
|
made.policy.redirectTo, _ = c.Values["redirect"].(string)
|
|
|
|
if named, carried := c.Values["auth"].(string); carried && strings.TrimSpace(named) != "" {
|
|
// **A declaration names a secret; it never holds one** (ADR 0108). Refused rather than
|
|
// tolerated, and the whole rule is dropped rather than served unprotected — the
|
|
// rejected option cannot come back by accident, which is the failure this check exists
|
|
// to make impossible.
|
|
if looksLikeACredential(named) {
|
|
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
|
|
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
|
|
c.From, c.Node, name)
|
|
continue
|
|
}
|
|
users, err := usersFrom(named)
|
|
if err != nil {
|
|
// Fail closed: the rule is kept so the name stays routed and answers, and it
|
|
// answers by refusing. Dropping it instead would make the name 404 and read as a
|
|
// withdrawn route rather than an unreadable secret.
|
|
made.policy.sealed = err.Error()
|
|
}
|
|
made.policy.users = users
|
|
}
|
|
|
|
// Only a rule that actually proxies needs somewhere to send the request.
|
|
if !made.policy.deny && made.policy.redirectTo == "" {
|
|
port, ok := asPort(c.Values["port"])
|
|
if !ok {
|
|
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
|
|
c.From, c.Node, name)
|
|
continue
|
|
}
|
|
// Where the mesh says that machine is. Empty means it is this one — a workload beside
|
|
// the proxy is ordinary, and reaching it over loopback is both correct and the only
|
|
// thing that works when there is no private network.
|
|
at := c.At
|
|
if at == "" {
|
|
at = "127.0.0.1"
|
|
}
|
|
// http unless the contribution says otherwise. A backend that terminates its own TLS
|
|
// with a certificate this proxy has no reason to trust — Mailu's webmail front is the
|
|
// first of these — is the reason `insecure` exists, and it stays the exception: every
|
|
// other target the mesh hands this proxy is a plain workload on the private network.
|
|
scheme, _ := c.Values["scheme"].(string)
|
|
scheme = strings.ToLower(strings.TrimSpace(scheme))
|
|
if scheme == "" {
|
|
scheme = "http"
|
|
}
|
|
if scheme != "http" && scheme != "https" {
|
|
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
|
|
"nor https; skipped", c.From, c.Node, name, scheme)
|
|
continue
|
|
}
|
|
made.insecure, _ = c.Values["insecure"].(bool)
|
|
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
|
|
}
|
|
|
|
out[host] = append(out[host], made)
|
|
|
|
// The internal-network alias, the same rule under a second host — a predecessor proxy
|
|
// answered both for one route, as a convenience (reaching a service over the VPN without a
|
|
// public TLS round trip), not as an access boundary; composing it here restores exactly
|
|
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
|
|
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name
|
|
// already has.
|
|
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" {
|
|
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
|
|
}
|
|
}
|
|
return out, public, nil
|
|
}
|
|
|
|
// asWhole is any whole number the mesh wrote, whatever its magnitude.
|
|
//
|
|
// **Not asPort.** Priority was read with the port reader first, which caps at 65535 — so a rule
|
|
// declared at a priority above that silently became priority 0 and stopped shadowing the route it
|
|
// exists to shadow. The one real rule this has to reproduce is declared at 100000, so the bug was
|
|
// exactly load-bearing. A priority is an ordering, not a port: it has no range.
|
|
func asWhole(v any) (int, bool) {
|
|
switch n := v.(type) {
|
|
case float64:
|
|
// JSON makes a float of every number, so a non-integral one was not meant as a priority.
|
|
if n != float64(int(n)) {
|
|
return 0, false
|
|
}
|
|
return int(n), true
|
|
case int:
|
|
return n, true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// asPath is the path prefix a rule is scoped to, or "" for every path.
|
|
func asPath(v any) string {
|
|
p, _ := v.(string)
|
|
p = strings.TrimSpace(p)
|
|
if p == "" {
|
|
return ""
|
|
}
|
|
if !strings.HasPrefix(p, "/") {
|
|
p = "/" + p
|
|
}
|
|
return p
|
|
}
|
|
|
|
// looksLikeACredential is the check that keeps a secret out of a declaration.
|
|
//
|
|
// It errs towards refusing: a value holding a `:` (the htpasswd separator) or opening with a bcrypt
|
|
// identifier is a credential, not a path, and no filesystem path the mesh writes needs either. A
|
|
// false refusal is a loud log and a route that does not serve; a false accept is a credential
|
|
// committed to a declaration, which is the thing being prevented.
|
|
func looksLikeACredential(v string) bool {
|
|
v = strings.TrimSpace(v)
|
|
return strings.Contains(v, ":") || strings.HasPrefix(v, "$2")
|
|
}
|
|
|
|
// usersFrom reads the credentials the mesh mounted, in the one format every htpasswd already is.
|
|
func usersFrom(path string) (map[string]string, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot read the secret named for this route: %w", err)
|
|
}
|
|
users := map[string]string{}
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
user, hash, ok := strings.Cut(line, ":")
|
|
if !ok || user == "" || hash == "" {
|
|
continue
|
|
}
|
|
users[user] = hash
|
|
}
|
|
if len(users) == 0 {
|
|
return nil, fmt.Errorf("the secret named for this route holds no usable credentials")
|
|
}
|
|
return users, nil
|
|
}
|
|
|
|
// asPort accepts what JSON makes of a number, which is a float even when it was written 8080.
|
|
func asPort(v any) (int, bool) {
|
|
switch n := v.(type) {
|
|
case float64:
|
|
if n < 1 || n > 65535 {
|
|
return 0, false
|
|
}
|
|
return int(n), true
|
|
case int:
|
|
if n < 1 || n > 65535 {
|
|
return 0, false
|
|
}
|
|
return n, true
|
|
}
|
|
return 0, false
|
|
}
|