diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go index 484ce2a..09c4a04 100644 --- a/examples/route-proxy/main.go +++ b/examples/route-proxy/main.go @@ -54,6 +54,7 @@ import ( "net" "net/http" "net/http/httputil" + "net/netip" "net/url" "os" "path/filepath" @@ -196,6 +197,96 @@ type table struct { // pass ACME's own validation (it has no public DNS to prove it against), so asking for it is // not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent. public map[string]bool + // inside is the private network's range, where a request must come from to be served a name + // that is only internal. Set once at start, never replaced with the routes: it is what the + // private network is, not what is routed on it. + inside sources + // bridges is the machine's own container networks, read from its interfaces and refreshed with + // the routes, since a compose network can appear at any time. + bridges sources +} + +// sources is the private network, as address ranges. The machine itself is always inside it — +// anything on a machine may call anything on it (novox/hq ADR 0144) — so loopback needs no range. +type sources []netip.Prefix + +// sourcesFrom reads the ranges the mesh wrote, separated by commas or spaces. A range that does not +// parse is an error, not a range skipped: the proxy would otherwise serve internal names to fewer +// machines than the mesh said, or start believing a typo. +func sourcesFrom(text string) (sources, error) { + var out sources + for _, field := range strings.FieldsFunc(text, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' }) { + prefix, err := netip.ParsePrefix(field) + if err != nil { + return nil, fmt.Errorf("%q is not an address range: %w", field, err) + } + out = append(out, prefix.Masked()) + } + return out, nil +} + +// bridgesFrom is the address ranges of the machine's container bridges — the same interfaces the +// mesh's guard names as the machine itself (docker0, and the br-* a compose network gets), so the +// proxy and the guard agree on what "this machine" is (novox/hq ADR 0144). +func bridgesFrom(interfaces map[string][]net.Addr) sources { + var out sources + for name, addrs := range interfaces { + if name != "docker0" && !strings.HasPrefix(name, "br-") { + continue + } + for _, a := range addrs { + if ipnet, ok := a.(*net.IPNet); ok { + if prefix, err := netip.ParsePrefix(ipnet.String()); err == nil { + out = append(out, prefix.Masked()) + } + } + } + } + return out +} + +// theseBridges reads this machine's interfaces for bridgesFrom. An interface that cannot be read +// contributes nothing: fewer callers inside, never more. +func theseBridges() sources { + interfaces, err := net.Interfaces() + if err != nil { + return nil + } + named := map[string][]net.Addr{} + for _, i := range interfaces { + if addrs, err := i.Addrs(); err == nil { + named[i.Name] = addrs + } + } + return bridgesFrom(named) +} + +// holds says whether a request from this remote address came from inside these ranges, or from +// the machine itself. +// +// **By source, which the guard deliberately is not** — it names interfaces because a source +// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request +// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a +// mesh or container address leave by the tunnel or a local bridge, never back to the claimant. +func (s sources) holds(remote string) bool { + host := remote + if h, _, err := net.SplitHostPort(remote); err == nil { + host = h + } + addr, err := netip.ParseAddr(host) + if err != nil { + return false + } + addr = addr.Unmap() + if addr.IsLoopback() { + return true + } + for _, prefix := range s { + if prefix.Contains(addr) { + return true + } + } + return false } func (t *table) set(routes map[string][]rule, public map[string]bool) { @@ -314,6 +405,41 @@ func bareHost(host string) string { return strings.ToLower(host) } +// hiddenFrom says whether this host must look unrouted to a request from this address: it is +// only an internal name, and the request did not come from the private network. +// +// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true** +// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from +// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does. +// Answered exactly as a name that was never routed, so an outsider learns nothing from asking. +func (t *table) hiddenFrom(host, remote string) bool { + if !t.eligibleForInternalACME(host) { + return false + } + t.mu.RLock() + defer t.mu.RUnlock() + return !t.inside.holds(remote) && !t.bridges.holds(remote) +} + +// setBridges replaces the machine's container networks. +func (t *table) setBridges(bridges sources) { + t.mu.Lock() + t.bridges = bridges + t.mu.Unlock() +} + +// namesSeenFrom is what this proxy says it serves to a request from this address — every routed +// name, less the internal-only ones when the request came from outside. +func (t *table) namesSeenFrom(remote string) []string { + out := []string{} + for _, name := range t.names() { + if !t.hiddenFrom(name, remote) { + out = append(out, name) + } + } + return out +} + func (t *table) names() []string { t.mu.RLock() defer t.mu.RUnlock() @@ -343,6 +469,18 @@ func run() error { } held := newTable() + // Unset means only this machine and its containers are inside, which serves an internal-only + // name to nobody else — refused rather than served to everyone, which is what the proxy did + // before it knew. + inside, err := sourcesFrom(os.Getenv("INTERNAL_SOURCES")) + if err != nil { + return fmt.Errorf("INTERNAL_SOURCES: %w", err) + } + if len(inside) == 0 { + log.Printf("INTERNAL_SOURCES is not set: a name that is only internal is served to this machine " + + "and its containers alone") + } + held.inside = inside read := func() { routes, public, err := routesFrom(path) if err != nil { @@ -353,6 +491,7 @@ func run() error { return } held.set(routes, public) + held.setBridges(theseBridges()) log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", ")) } read() @@ -422,19 +561,7 @@ func run() error { }() tlsConfig := publicManager.TLSConfig() - if internalManager != nil { - // Dispatched by which authority may certify this name at all — the same question - // eligibleForInternalACME already answers, asked once more at handshake time rather than - // only when an order is placed, since a cached certificate is served here on every request - // and never goes through HostPolicy again. - fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate - tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - if held.eligibleForInternalACME(hello.ServerName) { - return fromInternal(hello) - } - return fromPublic(hello) - } - } + tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager) server := &http.Server{ Addr: secure, @@ -592,6 +719,33 @@ func forThisAuthority(cache, directory string, root []byte) string { return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) } +// certificateFor picks the certificate a handshake is answered with. +// +// Dispatched by which authority may certify this name at all — the same question +// eligibleForInternalACME already answers, asked once more at handshake time rather than only when +// an order is placed, since a cached certificate is served here on every request and never goes +// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the +// private network asking for a name that is only internal: the certificate would name it. +func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error), + internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error) + if internalManager != nil { + fromInternal = internalManager.TLSConfig().GetCertificate + } + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if held.eligibleForInternalACME(hello.ServerName) { + if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) { + return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", + hello.ServerName) + } + if fromInternal != nil { + return fromInternal(hello) + } + } + return fromPublic(hello) + } +} + // newTable is an empty routing table. func newTable() *table { return &table{to: map[string][]rule{}} @@ -600,8 +754,9 @@ func newTable() *table { // handler is the proxy itself, separated so it can be driven by a test without a listener. func handler(held *table) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hidden := held.hiddenFrom(r.Host, r.RemoteAddr) matched, known := held.find(r.Host, r.URL.Path) - if !known { + if hidden || !known { // **Named, not a bare 404.** A route that was withdrawn and a name that never existed // are different things, and a proxy that says only "not found" makes an operator go // and read the mesh to tell them apart. What it is serving is the answer to both. @@ -611,13 +766,13 @@ func handler(held *table) http.Handler { // contradiction an operator would have to disbelieve the proxy to get past. w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusNotFound) - if held.routed(r.Host) { + if !hidden && held.routed(r.Host) { fmt.Fprintf(w, "%s is served here, but no route covers %q.\n", bareHost(r.Host), r.URL.Path) return } fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", - r.Host, strings.Join(held.names(), ", ")) + r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", ")) return } diff --git a/examples/route-proxy/reach_test.go b/examples/route-proxy/reach_test.go new file mode 100644 index 0000000..44f726c --- /dev/null +++ b/examples/route-proxy/reach_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +// behind is a workload the proxy can send to, and a table routing one public name and one +// internal-only name to it, with the private network set to inside. +func behind(t *testing.T, inside string) *table { + t.Helper() + workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "the workload") + })) + t.Cleanup(workload.Close) + at, _ := url.Parse(workload.URL) + host, port, _ := net.SplitHostPort(at.Host) + + routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[ + {"from":"app","node":"anchor","at":%q, + "values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}}, + {"from":"admin","node":"anchor","at":%q, + "values":{"internal-name":"admin.anchor.internal","port":%s}} + ]}`, host, port, host, port))) + if err != nil { + t.Fatal(err) + } + held := newTable() + held.inside, err = sourcesFrom(inside) + if err != nil { + t.Fatal(err) + } + held.set(routes, public) + return held +} + +// askFrom is what the proxy answers a request for host coming from remote. +func askFrom(held *table, host, remote string) (int, string) { + r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil) + r.RemoteAddr = remote + w := httptest.NewRecorder() + handler(held).ServeHTTP(w, r) + return w.Code, w.Body.String() +} + +// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR +// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name +// being internal kept nobody out: a request from the internet only had to carry it. +func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) { + held := behind(t, "10.10.0.0/24") + + if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK || + body != "the workload" { + t.Errorf("a request from the private network was not served: %d %q", code, body) + } + if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK { + t.Errorf("a request from the machine itself was not served: %d %q", code, body) + } + + code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000") + if code != http.StatusNotFound { + t.Fatalf("a request from outside the private network reached an internal-only name: %d %q", + code, body) + } + // Answered as a name never routed, and the list of what is served does not name it either — + // otherwise the refusal would tell an outsider exactly what to ask for from inside. + if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") { + t.Errorf("the refusal names the internal-only route to an outsider: %q", body) + } + if !strings.Contains(body, "app.example") { + t.Errorf("the refusal stopped listing the public names: %q", body) + } +} + +// The internal name of a route that also has a public one is internal too: served inside, and to +// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a +// name stays one thing whichever route it came from. +func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) { + held := behind(t, "10.10.0.0/24") + if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK { + t.Errorf("the internal alias stopped answering the private network: %d %q", code, body) + } + if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound { + t.Errorf("the internal alias was served to an outsider: %d", code) + } + if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK { + t.Errorf("the public name was refused to an outsider: %d", code) + } +} + +// A container on this machine reaches the proxy from its bridge's range, and is the machine itself +// (novox/hq ADR 0144): inside, though it is neither loopback nor the mesh. +func TestAContainerOnThisMachineIsInside(t *testing.T) { + held := behind(t, "10.10.0.0/24") + _, bridge, _ := net.ParseCIDR("172.18.0.1/16") + bridge.IP = net.ParseIP("172.18.0.1") + _, other, _ := net.ParseCIDR("192.168.1.20/24") + other.IP = net.ParseIP("192.168.1.20") + held.setBridges(bridgesFrom(map[string][]net.Addr{ + "br-0123456789ab": {bridge}, + "eth0": {other}, + })) + + if code, _ := askFrom(held, "admin.anchor.internal", "172.18.0.5:51000"); code != http.StatusOK { + t.Errorf("a container on this machine was refused: %d", code) + } + // The machine's own network is not a container bridge: a neighbour there is not the machine. + if code, _ := askFrom(held, "admin.anchor.internal", "192.168.1.30:51000"); code != http.StatusNotFound { + t.Errorf("a neighbour on the machine's network was served an internal-only name: %d", code) + } +} + +// With no private network said, only the machine itself is inside — refused to everyone else, +// never served to everyone. +func TestWithNoPrivateNetworkSaidAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) { + held := behind(t, "") + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound { + t.Errorf("an internal-only name was served with no private network said: %d", code) + } + if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK { + t.Errorf("an internal-only name was refused to the machine itself: %d", code) + } +} + +type from struct { + net.Conn + remote net.Addr +} + +func (c from) RemoteAddr() net.Addr { return c.remote } + +// The handshake refuses an internal-only name to an outsider too: the certificate would name it, +// and serving it would answer the question the routing refuses to. +func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) { + held := behind(t, "10.10.0.0/24") + served := &tls.Certificate{} + pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil) + hello := func(name, remote string) *tls.ClientHelloInfo { + addr, _ := net.ResolveTCPAddr("tcp", remote) + return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}} + } + + if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil { + t.Error("an outsider was handed a certificate for an internal-only name") + } + if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served { + t.Errorf("a client on the private network was refused: %v", err) + } + if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served { + t.Errorf("a public name was refused to an outsider: %v", err) + } +} + +// A range the proxy cannot read stops it, rather than serving internal names to fewer machines +// than the mesh said, or to a typo. +func TestAPrivateNetworkThatDoesNotParseIsRefused(t *testing.T) { + if _, err := sourcesFrom("10.10.0.0/24, not-a-range"); err == nil { + t.Error("a range that does not parse was accepted") + } + inside, err := sourcesFrom("10.10.0.0/24 fd00::/8") + if err != nil { + t.Fatal(err) + } + for remote, want := range map[string]bool{ + "10.10.0.200:1": true, + "[::ffff:10.10.0.3]:1": true, + "[fd00::1]:1": true, + "10.11.0.1:1": false, + "192.168.1.10:1": false, + "not-an-address": false, + } { + if inside.holds(remote) != want { + t.Errorf("%s inside the private network: got %v, want %v", remote, !want, want) + } + } +}