A route is a grant, and a provider is told where its consumer is

novox/hq 08-connectivity §3, built. The mirror of a database grant: there the consumer supplies a name and receives credentials; here it supplies a target and receives a name. Nothing new in the vocabulary — a route is a provision like any other. One field was missing and it is the one that matters for anything reaching back: a contribution now carries where the mesh says that machine is. A reverse proxy is told to send traffic to a consumer and has to open a connection, so without it every provider implementing a provision would have to know how the mesh names machines — a convention leaking into every module. The proxy itself is an example, not part of the control plane: the contract is the file, not this program. It replaces its table whole rather than merging, because the file is the whole truth about who has a route and merging would keep serving a name whose module was unassigned — the stale-route fault 08-connectivity lists as open, reintroduced one level down. A name it does not serve is refused by saying which it does: a route withdrawn and a name that never existed are different things.
2026-08-31 02:43:19 +02:00
parent ebcfd37b92
commit d0c0ee8dab
6 changed files with 407 additions and 2 deletions
@@ -53,6 +53,15 @@ provisioner-image:
 	@echo
 	@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'

+# The proxy that turns a route grant into traffic reaching a workload.
+PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
+PROXY_DEV_TAG ?= mesh-route-proxy:development
+
+proxy-image:
+	docker build -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
+	@echo
+	@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'
+
 # The whole gate. Raises a database, runs everything against it, and takes it down again --
 # including when the tests fail, which is why the teardown is not conditional.
 check: fmt vet postgres
@@ -1415,6 +1415,17 @@ func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]ca
 		return nil, err
 	}

+	// Where each consumer is, so a provider that must reach back to one does not have to know how
+	// the mesh names machines.
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return nil, err
+	}
+	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
+	if err != nil {
+		return nil, err
+	}
+
 	// What each consumer actually asked for, taken from that machine's own resolution rather than
 	// from a record beside it. A provider told to create a password and not what to create it for
 	// can do nothing with it, and the name a consumer wants is the consumer's to say.
@@ -1432,7 +1443,7 @@ func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]ca
 			return nil, err
 		}
 		out = append(out, catalogue.Grant{
-			Provision: s.Name, Consumer: s.Consumer,
+			Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
 			From: from, Values: values, Sealed: s.ForProvider})
 	}
 	return out, nil
@@ -0,0 +1,14 @@
+# The route proxy, as a module ships one.
+#
+# Static and FROM scratch like the control plane's image, and for the same reason: it is fetched
+# by digest and run on a machine, so everything in it is something a person would have to audit.
+FROM golang:1.25-alpine AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-route-proxy ./examples/route-proxy
+
+FROM scratch
+COPY --from=build /mesh-route-proxy /mesh-route-proxy
+ENTRYPOINT ["/mesh-route-proxy"]
@@ -0,0 +1,214 @@
+// A reverse proxy, in the form the mesh expects one.
+//
+// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable
+// declares it requires `route` and contributes the name it wants; the proxy provides `route` and
+// receives every consumer that asked. It is the mirror of a database grant: there the consumer
+// supplies a name and receives credentials, here it supplies a target and receives a name.
+//
+// **It is an example, not part of the control plane.** The control plane decides and never touches
+// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this
+// program. What lives here is that contract, written as something that runs so it can be read
+// rather than described.
+//
+// What it is given, written by the host from an ordinary declaration:
+//
+//	$ROUTES    every consumer, the name it asked for, and where the mesh says that machine is
+//
+// It re-reads on change rather than being restarted, for the same reason the provisioner does:
+// a route arriving or leaving is an ordinary event and must not drop the connections of every
+// other workload.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// what the mesh writes: the contributions file, one entry per consumer.
+type given struct {
+	Given []contribution `json:"given"`
+}
+
+type contribution struct {
+	From string `json:"from"`
+	Node string `json:"node"`
+	// At is where that machine is on the private network. The mesh knows it; a proxy that had to
+	// build it from the node name would be a naming convention copied into every provider.
+	At     string         `json:"at"`
+	Values map[string]any `json:"values"`
+}
+
+// table is what the proxy is currently serving, replaced whole whenever the file changes.
+//
+// Replaced rather than merged: the file is the whole truth about who has a route, so merging
+// would keep serving a name whose module was unassigned — which is the stale-route fault
+// 08-connectivity lists as open, reintroduced one level down.
+type table struct {
+	mu      sync.RWMutex
+	to      map[string]*httputil.ReverseProxy
+	targets map[string]string
+}
+
+func (t *table) set(routes map[string]string) {
+	made := map[string]*httputil.ReverseProxy{}
+	for name, target := range routes {
+		where, err := url.Parse(target)
+		if err != nil {
+			log.Printf("route %s points at %q, which is not a URL: %v", name, target, err)
+			continue
+		}
+		made[name] = httputil.NewSingleHostReverseProxy(where)
+	}
+	t.mu.Lock()
+	t.to, t.targets = made, routes
+	t.mu.Unlock()
+}
+
+func (t *table) find(host string) (*httputil.ReverseProxy, bool) {
+	// The port is not part of the name. A request to app.example:8080 is for app.example.
+	if h, _, err := net.SplitHostPort(host); err == nil {
+		host = h
+	}
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	p, ok := t.to[strings.ToLower(host)]
+	return p, ok
+}
+
+func (t *table) names() []string {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	out := make([]string, 0, len(t.targets))
+	for name := range t.targets {
+		out = append(out, name)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, "mesh-route-proxy: "+err.Error())
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	path := strings.TrimSpace(os.Getenv("ROUTES"))
+	if path == "" {
+		return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing")
+	}
+	listen := strings.TrimSpace(os.Getenv("LISTEN"))
+	if listen == "" {
+		listen = ":80"
+	}
+
+	held := newTable()
+	read := func() {
+		routes, err := routesFrom(path)
+		if err != nil {
+			// Kept serving what it had. A file being rewritten is momentarily unreadable, and
+			// dropping every route because one read landed mid-write would turn an ordinary
+			// event into an outage.
+			log.Printf("cannot read %s, keeping what is already served: %v", path, err)
+			return
+		}
+		held.set(routes)
+		log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", "))
+	}
+	read()
+
+	go func() {
+		for range time.Tick(2 * time.Second) {
+			read()
+		}
+	}()
+
+	return http.ListenAndServe(listen, handler(held))
+}
+
+// newTable is an empty routing table.
+func newTable() *table {
+	return &table{to: map[string]*httputil.ReverseProxy{}, targets: map[string]string{}}
+}
+
+// handler is the proxy itself, separated so it can be driven by a test without a listener.
+func handler(held *table) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		proxy, known := held.find(r.Host)
+		if !known {
+			// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
+			// are different things, and a proxy that says only "not found" makes an operator go
+			// and read the mesh to tell them apart. What it is serving is the answer to both.
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusNotFound)
+			fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
+				r.Host, strings.Join(held.names(), ", "))
+			return
+		}
+		proxy.ServeHTTP(w, r)
+	})
+}
+
+// routesFrom reads what the mesh wrote and turns it into name → target.
+func routesFrom(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var said given
+	if err := json.Unmarshal(raw, &said); err != nil {
+		return nil, err
+	}
+
+	out := map[string]string{}
+	for _, c := range said.Given {
+		name, _ := c.Values["name"].(string)
+		if name == "" {
+			log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
+			continue
+		}
+		port, ok := asPort(c.Values["port"])
+		if !ok {
+			log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
+				c.From, c.Node, name)
+			continue
+		}
+		// Where the mesh says that machine is. Empty means it is this one — a workload beside the
+		// proxy is ordinary, and reaching it over loopback is both correct and the only thing
+		// that works when there is no private network.
+		at := c.At
+		if at == "" {
+			at = "127.0.0.1"
+		}
+		out[strings.ToLower(name)] = fmt.Sprintf("http://%s:%d", at, port)
+	}
+	return out, nil
+}
+
+// asPort accepts what JSON makes of a number, which is a float even when it was written 8080.
+func asPort(v any) (int, bool) {
+	switch n := v.(type) {
+	case float64:
+		if n < 1 || n > 65535 {
+			return 0, false
+		}
+		return int(n), true
+	case int:
+		if n < 1 || n > 65535 {
+			return 0, false
+		}
+		return n, true
+	}
+	return 0, false
+}
@@ -0,0 +1,143 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func write(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "routes.json")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+// A route is a grant: the consumer supplies a target, and where that machine is comes from the
+// mesh rather than from a naming convention the proxy has to know.
+func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
+	routes, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[
+	  {"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Lower-cased, because a Host header is not case-sensitive and a route that only answers the
+	// spelling in the manifest answers half the requests made to it.
+	if routes["app.example"] != "http://laptop.internal:8080" {
+		t.Fatalf("the route does not point at the consumer: %v", routes)
+	}
+}
+
+// A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the
+// only thing that works when there is no private network.
+func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
+	routes, err := routesFrom(write(t, `{"given":[
+	  {"from":"app","node":"anchor","values":{"name":"app.example","port":9000}}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if routes["app.example"] != "http://127.0.0.1:9000" {
+		t.Fatalf("a workload on this machine was not reachable: %v", routes)
+	}
+}
+
+// Skipped rather than served wrongly. A route with no port would proxy to :0.
+func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
+	routes, err := routesFrom(write(t, `{"given":[
+	  {"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
+	  {"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
+	  {"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(routes) != 1 || routes["fine.example"] == "" {
+		t.Fatalf("an unusable contribution was served: %v", routes)
+	}
+}
+
+// End to end through the proxy itself: a request for the name reaches the workload, and a name
+// nobody asked for is refused in a way that says what IS served.
+func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
+	workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("the workload"))
+	}))
+	defer workload.Close()
+	target := strings.TrimPrefix(workload.URL, "http://")
+	host, port, _ := strings.Cut(target, ":")
+
+	held := newTable()
+	held.set(map[string]string{"app.example": "http://" + host + ":" + port})
+
+	proxy := httptest.NewServer(handler(held))
+	defer proxy.Close()
+
+	asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	asked.Host = "app.example"
+	answer, err := http.DefaultClient.Do(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer answer.Body.Close()
+	if answer.StatusCode != http.StatusOK {
+		t.Fatalf("a request for a served name got %d", answer.StatusCode)
+	}
+
+	// And a name that is not served says which are — a route withdrawn and a name that never
+	// existed are different things, and a bare 404 makes an operator go and read the mesh.
+	other, _ := http.NewRequest(http.MethodGet, proxy.URL, nil)
+	other.Host = "nobody.example"
+	refused, err := http.DefaultClient.Do(other)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer refused.Body.Close()
+	if refused.StatusCode != http.StatusNotFound {
+		t.Fatalf("a name nobody asked for got %d", refused.StatusCode)
+	}
+	body := make([]byte, 256)
+	n, _ := refused.Body.Read(body)
+	if !strings.Contains(string(body[:n]), "app.example") {
+		t.Fatalf("the refusal does not say what is served: %s", body[:n])
+	}
+}
+
+// The file is the whole truth about who has a route, so the table replaces rather than merges.
+//
+// Merging would keep serving a name whose module was unassigned — the stale-route fault
+// 08-connectivity lists as open, reintroduced one level down. A stale public name pointing at
+// nothing fails more visibly than a stale grant, which is exactly why it must not survive.
+func TestWithdrawingARouteStopsServingIt(t *testing.T) {
+	held := newTable()
+	held.set(map[string]string{
+		"going.example":   "http://a.internal:80",
+		"staying.example": "http://b.internal:80",
+	})
+	held.set(map[string]string{"staying.example": "http://b.internal:80"})
+
+	if _, still := held.find("going.example"); still {
+		t.Fatal("a route whose module was unassigned is still served")
+	}
+	if _, kept := held.find("staying.example"); !kept {
+		t.Fatal("withdrawing one route took another with it")
+	}
+}
+
+// A Host header carries a port and the name does not.
+func TestARequestNamingAPortStillFindsItsRoute(t *testing.T) {
+	held := newTable()
+	held.set(map[string]string{"app.example": "http://a.internal:8080"})
+	if _, found := held.find("app.example:8080"); !found {
+		t.Fatal("a request to app.example:8080 did not find the route for app.example")
+	}
+}
@@ -38,6 +38,12 @@ type Grant struct {
 	// Values are what that module contributed — the name it wants, and anything else the
 	// provision's own vocabulary defines.
 	Values map[string]any
+	// At is where the consuming machine is on the private network, empty if it is not on one.
+	//
+	// Passed in with the grant because it is a fact about another machine, and resolution answers
+	// questions about one. A provider that must reach back to its consumer — a reverse proxy is
+	// the whole reason this exists — otherwise has to know how the mesh names machines.
+	At string
 	// Sealed is the credential, closed to the providing node.
 	Sealed string
 }
@@ -284,6 +290,14 @@ type Contribution struct {
 	// cannot do anything with it. Contributions were node-local until this, which meant the one
 	// case that most needed them was the one they did not reach.
 	Node string `json:"node,omitempty"`
+	// At is where that machine is on the private network, empty when it is not on one or when it
+	// is this machine.
+	//
+	// The mesh knows it and a provider should not have to derive it. A reverse proxy is told
+	// *send traffic to this consumer* and has to open a connection — so without this every
+	// provider that reaches back to a consumer would have to know how the mesh names machines,
+	// which is a convention leaking into every module that implements a provision.
+	At string `json:"at,omitempty"`
 	// Secret is the file on this machine holding that consumer's credential, sealed to it.
 	//
 	// Named rather than carried, for the same reason the private network's key is: the mesh
@@ -331,7 +345,7 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 			continue
 		}
 		out[g.Provision] = append(out[g.Provision], Contribution{
-			From: g.From, Node: g.Consumer, Values: g.Values,
+			From: g.From, Node: g.Consumer, At: g.At, Values: g.Values,
 			Secret: grantPath(directories[g.Provision], g.Consumer),
 		})
 	}