diff --git a/modules/route-proxy/cmd/route-proxy/bus.go b/modules/route-proxy/cmd/route-proxy/bus.go new file mode 100644 index 0000000..51ed510 --- /dev/null +++ b/modules/route-proxy/cmd/route-proxy/bus.go @@ -0,0 +1,132 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167). +// +// **The proxy is told, not left to work it out.** Its membership carries the routes it is given — +// the same contributions its file is written from — and every machine's address on the private +// network, which is who may be served an internal name. Read once at connect and followed, so a +// route added or a machine joining reaches a running proxy without a restart. + +// credential is the bus account the mesh delivered as this module's own secret named broker. +type credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint"` + Node string `json:"node"` + Module string `json:"module"` + User string `json:"user"` + Password string `json:"password"` +} + +// followMembership connects with the credential in path and applies every membership the mesh +// issues this proxy. It retries the first connection for as long as it takes: a proxy that started +// before the bus keeps serving the file, and takes the bus when it answers. +func followMembership(path string, held *table, fromBus *atomic.Bool) { + for { + err := followOnce(path, held, fromBus) + if err == nil { + return + } + log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err) + time.Sleep(30 * time.Second) + } +} + +func followOnce(path string, held *table, fromBus *atomic.Bool) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var cred credential + if err := json.Unmarshal(raw, &cred); err != nil { + return fmt.Errorf("the broker credential is not one: %w", err) + } + if cred.Node == "" || cred.Module == "" { + return fmt.Errorf("the broker credential names no node or module, so it has no membership") + } + + opts := []nats.Option{ + nats.Name(cred.Node + "." + cred.Module), + nats.UserInfo(cred.User, cred.Password), + // Its own inbox, and nothing wider: every principal is granted `_INBOX..>` alone. + nats.CustomInboxPrefix("_INBOX." + cred.User), + // The bus being restarted is an upgrade, not a reason to stop following. + nats.MaxReconnects(-1), + } + if strings.TrimSpace(cred.Fingerprint) != "" { + opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint))) + } + conn, err := nats.Connect(cred.URL, opts...) + if err != nil { + return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err) + } + + subject := broker.MembershipSubject(cred.Node, cred.Module) + apply := func(body []byte) { + var issued broker.Membership + if err := json.Unmarshal(body, &issued); err != nil { + log.Printf("a membership arrived that is not one: %v", err) + return + } + if took := applyMembership(issued, held); took && !fromBus.Swap(true) { + log.Printf("routes now come from this proxy's membership on %s", subject) + } + } + + // Followed first, read second: an issue landing between the two is applied, not missed. + if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil { + conn.Close() + return fmt.Errorf("cannot follow %s: %w", subject, err) + } + // The subject-addressed direct get: the one request this account may make of the stream. + got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second) + switch { + case err != nil: + log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err) + case got.Header.Get("Status") != "" || len(got.Data) == 0: + log.Printf("no membership issued on %s yet; serving the file until one is", subject) + default: + apply(got.Data) + } + return nil +} + +// applyMembership serves what a membership says, and says whether it said anything about routes. +// +// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays +// the source rather than every route being withdrawn because a field was absent. +func applyMembership(issued broker.Membership, held *table) bool { + raw, carries := issued.Receives["route"] + if !carries { + return false + } + var contributions []contribution + if err := json.Unmarshal(raw, &contributions); err != nil { + log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err) + return false + } + inside, err := sourcesOf(issued.Mesh) + if err != nil { + log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err) + return false + } + routes, public := routesOf(contributions) + held.set(routes, public) + held.setInside(inside) + log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s", + len(routes), len(inside), strings.Join(held.names(), ", ")) + return true +} diff --git a/modules/route-proxy/cmd/route-proxy/main.go b/modules/route-proxy/cmd/route-proxy/main.go index 484ce2a..47d4f93 100644 --- a/modules/route-proxy/cmd/route-proxy/main.go +++ b/modules/route-proxy/cmd/route-proxy/main.go @@ -54,12 +54,14 @@ import ( "net" "net/http" "net/http/httputil" + "net/netip" "net/url" "os" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "time" "golang.org/x/crypto/acme" @@ -196,6 +198,63 @@ type table struct { // pass ACME's own validation (it has no public DNS to prove it against), so asking for it is // not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent. public map[string]bool + // inside is where a request must come from to be served a name that is only internal: every + // machine's address on the private network, as the mesh issued it in this proxy's membership + // (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside. + inside sources +} + +// sources is who may be served an internal name: the private network's addresses as the mesh +// issued them. The machine itself is always inside — anything on a machine may call anything on it +// (novox/hq ADR 0144) — so loopback needs no entry. +type sources []netip.Prefix + +// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does +// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to +// fewer machines than the mesh said, and say nothing. +func sourcesOf(mesh []string) (sources, error) { + var out sources + for _, entry := range mesh { + entry = strings.TrimSpace(entry) + if prefix, err := netip.ParsePrefix(entry); err == nil { + out = append(out, prefix.Masked()) + continue + } + addr, err := netip.ParseAddr(entry) + if err != nil { + return nil, fmt.Errorf("%q is not an address on the private network", entry) + } + addr = addr.Unmap() + out = append(out, netip.PrefixFrom(addr, addr.BitLen())) + } + return out, nil +} + +// holds says whether a request from this remote address came from the mesh or the machine itself. +// +// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source +// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request +// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a +// mesh address leave by the tunnel, never back to the claimant. +func (s sources) holds(remote string) bool { + host := remote + if h, _, err := net.SplitHostPort(remote); err == nil { + host = h + } + addr, err := netip.ParseAddr(host) + if err != nil { + return false + } + addr = addr.Unmap() + if addr.IsLoopback() { + return true + } + for _, prefix := range s { + if prefix.Contains(addr) { + return true + } + } + return false } func (t *table) set(routes map[string][]rule, public map[string]bool) { @@ -314,6 +373,41 @@ func bareHost(host string) string { return strings.ToLower(host) } +// hiddenFrom says whether this host must look unrouted to a request from this address: it is +// only an internal name, and the request did not come from the private network. +// +// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true** +// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from +// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does. +// Answered exactly as a name that was never routed, so an outsider learns nothing from asking. +func (t *table) hiddenFrom(host, remote string) bool { + if !t.eligibleForInternalACME(host) { + return false + } + t.mu.RLock() + defer t.mu.RUnlock() + return !t.inside.holds(remote) +} + +// setInside replaces who the mesh is, as the membership said. +func (t *table) setInside(inside sources) { + t.mu.Lock() + t.inside = inside + t.mu.Unlock() +} + +// namesSeenFrom is what this proxy says it serves to a request from this address — every routed +// name, less the internal-only ones when the request came from outside. +func (t *table) namesSeenFrom(remote string) []string { + out := []string{} + for _, name := range t.names() { + if !t.hiddenFrom(name, remote) { + out = append(out, name) + } + } + return out +} + func (t *table) names() []string { t.mu.RLock() defer t.mu.RUnlock() @@ -343,7 +437,20 @@ func run() error { } held := newTable() + // **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries + // the routes and who the mesh is; the file carries the routes alone, so while the proxy reads + // it an internal name is served to this machine and to nobody else — refused, never opened. + fromBus := &atomic.Bool{} + if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" { + go followMembership(credential, held, fromBus) + } else { + log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+ + "internal is served to this machine alone", path) + } read := func() { + if fromBus.Load() { + return + } routes, public, err := routesFrom(path) if err != nil { // Kept serving what it had. A file being rewritten is momentarily unreadable, and @@ -422,19 +529,7 @@ func run() error { }() tlsConfig := publicManager.TLSConfig() - if internalManager != nil { - // Dispatched by which authority may certify this name at all — the same question - // eligibleForInternalACME already answers, asked once more at handshake time rather than - // only when an order is placed, since a cached certificate is served here on every request - // and never goes through HostPolicy again. - fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate - tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - if held.eligibleForInternalACME(hello.ServerName) { - return fromInternal(hello) - } - return fromPublic(hello) - } - } + tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager) server := &http.Server{ Addr: secure, @@ -592,6 +687,33 @@ func forThisAuthority(cache, directory string, root []byte) string { return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) } +// certificateFor picks the certificate a handshake is answered with. +// +// Dispatched by which authority may certify this name at all — the same question +// eligibleForInternalACME already answers, asked once more at handshake time rather than only when +// an order is placed, since a cached certificate is served here on every request and never goes +// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the +// private network asking for a name that is only internal: the certificate would name it. +func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error), + internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error) + if internalManager != nil { + fromInternal = internalManager.TLSConfig().GetCertificate + } + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if held.eligibleForInternalACME(hello.ServerName) { + if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) { + return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", + hello.ServerName) + } + if fromInternal != nil { + return fromInternal(hello) + } + } + return fromPublic(hello) + } +} + // newTable is an empty routing table. func newTable() *table { return &table{to: map[string][]rule{}} @@ -600,8 +722,9 @@ func newTable() *table { // handler is the proxy itself, separated so it can be driven by a test without a listener. func handler(held *table) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hidden := held.hiddenFrom(r.Host, r.RemoteAddr) matched, known := held.find(r.Host, r.URL.Path) - if !known { + if hidden || !known { // **Named, not a bare 404.** A route that was withdrawn and a name that never existed // are different things, and a proxy that says only "not found" makes an operator go // and read the mesh to tell them apart. What it is serving is the answer to both. @@ -611,13 +734,13 @@ func handler(held *table) http.Handler { // contradiction an operator would have to disbelieve the proxy to get past. w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusNotFound) - if held.routed(r.Host) { + if !hidden && held.routed(r.Host) { fmt.Fprintf(w, "%s is served here, but no route covers %q.\n", bareHost(r.Host), r.URL.Path) return } fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", - r.Host, strings.Join(held.names(), ", ")) + r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", ")) return } @@ -731,10 +854,16 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { if err := json.Unmarshal(raw, &said); err != nil { return nil, nil, err } + routes, public := routesOf(said.Given) + return routes, public, nil +} +// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public +// names — the same whether the contributions came in the file or in the membership. +func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) { out := map[string][]rule{} public := map[string]bool{} - for _, c := range said.Given { + for _, c := range contributions { name, _ := c.Values["name"].(string) name = strings.TrimSpace(name) internal, _ := c.Values["internal-name"].(string) @@ -839,7 +968,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made) } } - return out, public, nil + return out, public } // asWhole is any whole number the mesh wrote, whatever its magnitude. diff --git a/modules/route-proxy/cmd/route-proxy/reach_test.go b/modules/route-proxy/cmd/route-proxy/reach_test.go new file mode 100644 index 0000000..151a41a --- /dev/null +++ b/modules/route-proxy/cmd/route-proxy/reach_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +// behind is a workload the proxy can send to, and a table routing one public name and one +// internal-only name to it, with the mesh's machines as the membership would issue them. +func behind(t *testing.T, mesh ...string) *table { + t.Helper() + workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "the workload") + })) + t.Cleanup(workload.Close) + at, _ := url.Parse(workload.URL) + host, port, _ := net.SplitHostPort(at.Host) + + routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[ + {"from":"app","node":"anchor","at":%q, + "values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}}, + {"from":"admin","node":"anchor","at":%q, + "values":{"internal-name":"admin.anchor.internal","port":%s}} + ]}`, host, port, host, port))) + if err != nil { + t.Fatal(err) + } + held := newTable() + inside, err := sourcesOf(mesh) + if err != nil { + t.Fatal(err) + } + held.setInside(inside) + held.set(routes, public) + return held +} + +// askFrom is what the proxy answers a request for host coming from remote. +func askFrom(held *table, host, remote string) (int, string) { + r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil) + r.RemoteAddr = remote + w := httptest.NewRecorder() + handler(held).ServeHTTP(w, r) + return w.Code, w.Body.String() +} + +// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR +// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name +// being internal kept nobody out: a request from the internet only had to carry it. +func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + + if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK || + body != "the workload" { + t.Errorf("a request from the private network was not served: %d %q", code, body) + } + if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK { + t.Errorf("a request from the machine itself was not served: %d %q", code, body) + } + + code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000") + if code != http.StatusNotFound { + t.Fatalf("a request from outside the private network reached an internal-only name: %d %q", + code, body) + } + // Answered as a name never routed, and the list of what is served does not name it either — + // otherwise the refusal would tell an outsider exactly what to ask for from inside. + if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") { + t.Errorf("the refusal names the internal-only route to an outsider: %q", body) + } + if !strings.Contains(body, "app.example") { + t.Errorf("the refusal stopped listing the public names: %q", body) + } +} + +// The internal name of a route that also has a public one is internal too: served inside, and to +// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a +// name stays one thing whichever route it came from. +func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK { + t.Errorf("the internal alias stopped answering the private network: %d %q", code, body) + } + if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound { + t.Errorf("the internal alias was served to an outsider: %d", code) + } + if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK { + t.Errorf("the public name was refused to an outsider: %d", code) + } +} + +// Before a membership has said who the mesh is, only the machine itself is inside — refused to +// everyone else, never served to everyone. +func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) { + held := behind(t) + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound { + t.Errorf("an internal-only name was served with no private network said: %d", code) + } + if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK { + t.Errorf("an internal-only name was refused to the machine itself: %d", code) + } +} + +type from struct { + net.Conn + remote net.Addr +} + +func (c from) RemoteAddr() net.Addr { return c.remote } + +// The handshake refuses an internal-only name to an outsider too: the certificate would name it, +// and serving it would answer the question the routing refuses to. +func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + served := &tls.Certificate{} + pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil) + hello := func(name, remote string) *tls.ClientHelloInfo { + addr, _ := net.ResolveTCPAddr("tcp", remote) + return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}} + } + + if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil { + t.Error("an outsider was handed a certificate for an internal-only name") + } + if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served { + t.Errorf("a client on the private network was refused: %v", err) + } + if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served { + t.Errorf("a public name was refused to an outsider: %v", err) + } +} + +// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is +// refused rather than skipped, so a typo never quietly narrows or widens who is inside. +func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) { + if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil { + t.Error("an entry that is not an address was accepted") + } + inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"}) + if err != nil { + t.Fatal(err) + } + for remote, want := range map[string]bool{ + "10.10.0.1:1": true, + "[::ffff:10.10.0.1]:1": true, + "[fd00::1]:1": true, + "10.20.0.200:1": true, + "10.10.0.2:1": false, + "192.168.1.10:1": false, + "not-an-address": false, + } { + if inside.holds(remote) != want { + t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want) + } + } +} + +// What the mesh issues is what is served: the routes in the membership, internal names to the +// machines it names (novox/hq ADR 0167). +func TestAMembershipIsServedAsIssued(t *testing.T) { + held := newTable() + took := applyMembership(broker.Membership{ + Receives: map[string]json.RawMessage{"route": json.RawMessage(`[ + {"from":"admin","node":"anchor","at":"anchor.internal", + "values":{"internal-name":"admin.anchor.internal","port":8080}}]`)}, + Mesh: []string{"10.10.0.7"}, + }, held) + if !took { + t.Fatal("a membership carrying routes was not applied") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { + t.Error("a machine the membership names was refused the internal-only route") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound { + t.Errorf("a machine the membership does not name was served the internal-only route: %d", code) + } +} + +// A membership that says nothing about routes is one from a controller that does not issue them, +// and changes nothing: the file stays the source rather than every route being withdrawn. +func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) { + held := behind(t, "10.10.0.7") + before := held.names() + if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) { + t.Error("a membership without routes was taken as the source of routes") + } + if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") { + t.Errorf("a membership without routes changed what is served: %v, was %v", got, before) + } + if applyMembership(broker.Membership{ + Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)}, + Mesh: []string{"not-an-address"}, + }, held) { + t.Error("a membership whose mesh cannot be read was applied") + } +}