From d0c0ee8dab27b0ce3fb307072a42fbe62e8dfb74 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 31 Aug 2026 02:43:19 +0200 Subject: [PATCH] A route is a grant, and a provider is told where its consumer is MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit novox/hq 08-connectivity §3, built. The mirror of a database grant: there the consumer supplies a name and receives credentials; here it supplies a target and receives a name. Nothing new in the vocabulary — a route is a provision like any other. One field was missing and it is the one that matters for anything reaching back: a contribution now carries where the mesh says that machine is. A reverse proxy is told to send traffic to a consumer and has to open a connection, so without it every provider implementing a provision would have to know how the mesh names machines — a convention leaking into every module. The proxy itself is an example, not part of the control plane: the contract is the file, not this program. It replaces its table whole rather than merging, because the file is the whole truth about who has a route and merging would keep serving a name whose module was unassigned — the stale-route fault 08-connectivity lists as open, reintroduced one level down. A name it does not serve is refused by saying which it does: a route withdrawn and a name that never existed are different things. --- Makefile | 9 ++ cmd/mesh-control/main.go | 13 +- examples/route-proxy/Dockerfile | 14 ++ examples/route-proxy/main.go | 214 ++++++++++++++++++++++++++++ examples/route-proxy/routes_test.go | 143 +++++++++++++++++++ internal/catalogue/declaration.go | 16 ++- 6 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 examples/route-proxy/Dockerfile create mode 100644 examples/route-proxy/main.go create mode 100644 examples/route-proxy/routes_test.go diff --git a/Makefile b/Makefile index 3c53ef2..36ce819 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,15 @@ provisioner-image: @echo @docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' +# The proxy that turns a route grant into traffic reaching a workload. +PROXY_IMAGE ?= mesh-route-proxy:$(VERSION) +PROXY_DEV_TAG ?= mesh-route-proxy:development + +proxy-image: + docker build -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) . + @echo + @docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' + # The whole gate. Raises a database, runs everything against it, and takes it down again -- # including when the tests fail, which is why the teardown is not conditional. check: fmt vet postgres diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index 0368d7d..9abe1e1 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -1415,6 +1415,17 @@ func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]ca return nil, err } + // Where each consumer is, so a provider that must reach back to one does not have to know how + // the mesh names machines. + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, err + } + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return nil, err + } + // What each consumer actually asked for, taken from that machine's own resolution rather than // from a record beside it. A provider told to create a password and not what to create it for // can do nothing with it, and the name a consumer wants is the consumer's to say. @@ -1432,7 +1443,7 @@ func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]ca return nil, err } out = append(out, catalogue.Grant{ - Provision: s.Name, Consumer: s.Consumer, + Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer], From: from, Values: values, Sealed: s.ForProvider}) } return out, nil diff --git a/examples/route-proxy/Dockerfile b/examples/route-proxy/Dockerfile new file mode 100644 index 0000000..c807c3d --- /dev/null +++ b/examples/route-proxy/Dockerfile @@ -0,0 +1,14 @@ +# The route proxy, as a module ships one. +# +# Static and FROM scratch like the control plane's image, and for the same reason: it is fetched +# by digest and run on a machine, so everything in it is something a person would have to audit. +FROM golang:1.25-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-route-proxy ./examples/route-proxy + +FROM scratch +COPY --from=build /mesh-route-proxy /mesh-route-proxy +ENTRYPOINT ["/mesh-route-proxy"] diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go new file mode 100644 index 0000000..f50d322 --- /dev/null +++ b/examples/route-proxy/main.go @@ -0,0 +1,214 @@ +// A reverse proxy, in the form the mesh expects one. +// +// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable +// declares it requires `route` and contributes the name it wants; the proxy provides `route` and +// receives every consumer that asked. It is the mirror of a database grant: there the consumer +// supplies a name and receives credentials, here it supplies a target and receives a name. +// +// **It is an example, not part of the control plane.** The control plane decides and never touches +// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this +// program. What lives here is that contract, written as something that runs so it can be read +// rather than described. +// +// What it is given, written by the host from an ordinary declaration: +// +// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is +// +// It re-reads on change rather than being restarted, for the same reason the provisioner does: +// a route arriving or leaving is an ordinary event and must not drop the connections of every +// other workload. +package main + +import ( + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "sort" + "strings" + "sync" + "time" +) + +// what the mesh writes: the contributions file, one entry per consumer. +type given struct { + Given []contribution `json:"given"` +} + +type contribution struct { + From string `json:"from"` + Node string `json:"node"` + // At is where that machine is on the private network. The mesh knows it; a proxy that had to + // build it from the node name would be a naming convention copied into every provider. + At string `json:"at"` + Values map[string]any `json:"values"` +} + +// table is what the proxy is currently serving, replaced whole whenever the file changes. +// +// Replaced rather than merged: the file is the whole truth about who has a route, so merging +// would keep serving a name whose module was unassigned — which is the stale-route fault +// 08-connectivity lists as open, reintroduced one level down. +type table struct { + mu sync.RWMutex + to map[string]*httputil.ReverseProxy + targets map[string]string +} + +func (t *table) set(routes map[string]string) { + made := map[string]*httputil.ReverseProxy{} + for name, target := range routes { + where, err := url.Parse(target) + if err != nil { + log.Printf("route %s points at %q, which is not a URL: %v", name, target, err) + continue + } + made[name] = httputil.NewSingleHostReverseProxy(where) + } + t.mu.Lock() + t.to, t.targets = made, routes + t.mu.Unlock() +} + +func (t *table) find(host string) (*httputil.ReverseProxy, bool) { + // The port is not part of the name. A request to app.example:8080 is for app.example. + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + t.mu.RLock() + defer t.mu.RUnlock() + p, ok := t.to[strings.ToLower(host)] + return p, ok +} + +func (t *table) names() []string { + t.mu.RLock() + defer t.mu.RUnlock() + out := make([]string, 0, len(t.targets)) + for name := range t.targets { + out = append(out, name) + } + sort.Strings(out) + return out +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "mesh-route-proxy: "+err.Error()) + os.Exit(1) + } +} + +func run() error { + path := strings.TrimSpace(os.Getenv("ROUTES")) + if path == "" { + return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing") + } + listen := strings.TrimSpace(os.Getenv("LISTEN")) + if listen == "" { + listen = ":80" + } + + held := newTable() + read := func() { + routes, err := routesFrom(path) + if err != nil { + // Kept serving what it had. A file being rewritten is momentarily unreadable, and + // dropping every route because one read landed mid-write would turn an ordinary + // event into an outage. + log.Printf("cannot read %s, keeping what is already served: %v", path, err) + return + } + held.set(routes) + log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", ")) + } + read() + + go func() { + for range time.Tick(2 * time.Second) { + read() + } + }() + + return http.ListenAndServe(listen, handler(held)) +} + +// newTable is an empty routing table. +func newTable() *table { + return &table{to: map[string]*httputil.ReverseProxy{}, targets: map[string]string{}} +} + +// handler is the proxy itself, separated so it can be driven by a test without a listener. +func handler(held *table) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxy, known := held.find(r.Host) + if !known { + // **Named, not a bare 404.** A route that was withdrawn and a name that never existed + // are different things, and a proxy that says only "not found" makes an operator go + // and read the mesh to tell them apart. What it is serving is the answer to both. + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusNotFound) + fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", + r.Host, strings.Join(held.names(), ", ")) + return + } + proxy.ServeHTTP(w, r) + }) +} + +// routesFrom reads what the mesh wrote and turns it into name → target. +func routesFrom(path string) (map[string]string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var said given + if err := json.Unmarshal(raw, &said); err != nil { + return nil, err + } + + out := map[string]string{} + for _, c := range said.Given { + name, _ := c.Values["name"].(string) + if name == "" { + log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node) + continue + } + port, ok := asPort(c.Values["port"]) + if !ok { + log.Printf("%s on %s asked for route %q and gave no usable port; skipped", + c.From, c.Node, name) + continue + } + // Where the mesh says that machine is. Empty means it is this one — a workload beside the + // proxy is ordinary, and reaching it over loopback is both correct and the only thing + // that works when there is no private network. + at := c.At + if at == "" { + at = "127.0.0.1" + } + out[strings.ToLower(name)] = fmt.Sprintf("http://%s:%d", at, port) + } + return out, nil +} + +// asPort accepts what JSON makes of a number, which is a float even when it was written 8080. +func asPort(v any) (int, bool) { + switch n := v.(type) { + case float64: + if n < 1 || n > 65535 { + return 0, false + } + return int(n), true + case int: + if n < 1 || n > 65535 { + return 0, false + } + return n, true + } + return 0, false +} diff --git a/examples/route-proxy/routes_test.go b/examples/route-proxy/routes_test.go new file mode 100644 index 0000000..628ed7b --- /dev/null +++ b/examples/route-proxy/routes_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func write(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "routes.json") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// A route is a grant: the consumer supplies a target, and where that machine is comes from the +// mesh rather than from a naming convention the proxy has to know. +func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) { + routes, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[ + {"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}} + ]}`)) + if err != nil { + t.Fatal(err) + } + // Lower-cased, because a Host header is not case-sensitive and a route that only answers the + // spelling in the manifest answers half the requests made to it. + if routes["app.example"] != "http://laptop.internal:8080" { + t.Fatalf("the route does not point at the consumer: %v", routes) + } +} + +// A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the +// only thing that works when there is no private network. +func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { + routes, err := routesFrom(write(t, `{"given":[ + {"from":"app","node":"anchor","values":{"name":"app.example","port":9000}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if routes["app.example"] != "http://127.0.0.1:9000" { + t.Fatalf("a workload on this machine was not reachable: %v", routes) + } +} + +// Skipped rather than served wrongly. A route with no port would proxy to :0. +func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) { + routes, err := routesFrom(write(t, `{"given":[ + {"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}}, + {"from":"b","node":"n","at":"n.internal","values":{"port":8080}}, + {"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if len(routes) != 1 || routes["fine.example"] == "" { + t.Fatalf("an unusable contribution was served: %v", routes) + } +} + +// End to end through the proxy itself: a request for the name reaches the workload, and a name +// nobody asked for is refused in a way that says what IS served. +func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) { + workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("the workload")) + })) + defer workload.Close() + target := strings.TrimPrefix(workload.URL, "http://") + host, port, _ := strings.Cut(target, ":") + + held := newTable() + held.set(map[string]string{"app.example": "http://" + host + ":" + port}) + + proxy := httptest.NewServer(handler(held)) + defer proxy.Close() + + asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil) + if err != nil { + t.Fatal(err) + } + asked.Host = "app.example" + answer, err := http.DefaultClient.Do(asked) + if err != nil { + t.Fatal(err) + } + defer answer.Body.Close() + if answer.StatusCode != http.StatusOK { + t.Fatalf("a request for a served name got %d", answer.StatusCode) + } + + // And a name that is not served says which are — a route withdrawn and a name that never + // existed are different things, and a bare 404 makes an operator go and read the mesh. + other, _ := http.NewRequest(http.MethodGet, proxy.URL, nil) + other.Host = "nobody.example" + refused, err := http.DefaultClient.Do(other) + if err != nil { + t.Fatal(err) + } + defer refused.Body.Close() + if refused.StatusCode != http.StatusNotFound { + t.Fatalf("a name nobody asked for got %d", refused.StatusCode) + } + body := make([]byte, 256) + n, _ := refused.Body.Read(body) + if !strings.Contains(string(body[:n]), "app.example") { + t.Fatalf("the refusal does not say what is served: %s", body[:n]) + } +} + +// The file is the whole truth about who has a route, so the table replaces rather than merges. +// +// Merging would keep serving a name whose module was unassigned — the stale-route fault +// 08-connectivity lists as open, reintroduced one level down. A stale public name pointing at +// nothing fails more visibly than a stale grant, which is exactly why it must not survive. +func TestWithdrawingARouteStopsServingIt(t *testing.T) { + held := newTable() + held.set(map[string]string{ + "going.example": "http://a.internal:80", + "staying.example": "http://b.internal:80", + }) + held.set(map[string]string{"staying.example": "http://b.internal:80"}) + + if _, still := held.find("going.example"); still { + t.Fatal("a route whose module was unassigned is still served") + } + if _, kept := held.find("staying.example"); !kept { + t.Fatal("withdrawing one route took another with it") + } +} + +// A Host header carries a port and the name does not. +func TestARequestNamingAPortStillFindsItsRoute(t *testing.T) { + held := newTable() + held.set(map[string]string{"app.example": "http://a.internal:8080"}) + if _, found := held.find("app.example:8080"); !found { + t.Fatal("a request to app.example:8080 did not find the route for app.example") + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 6804040..1331b6d 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -38,6 +38,12 @@ type Grant struct { // Values are what that module contributed — the name it wants, and anything else the // provision's own vocabulary defines. Values map[string]any + // At is where the consuming machine is on the private network, empty if it is not on one. + // + // Passed in with the grant because it is a fact about another machine, and resolution answers + // questions about one. A provider that must reach back to its consumer — a reverse proxy is + // the whole reason this exists — otherwise has to know how the mesh names machines. + At string // Sealed is the credential, closed to the providing node. Sealed string } @@ -284,6 +290,14 @@ type Contribution struct { // cannot do anything with it. Contributions were node-local until this, which meant the one // case that most needed them was the one they did not reach. Node string `json:"node,omitempty"` + // At is where that machine is on the private network, empty when it is not on one or when it + // is this machine. + // + // The mesh knows it and a provider should not have to derive it. A reverse proxy is told + // *send traffic to this consumer* and has to open a connection — so without this every + // provider that reaches back to a consumer would have to know how the mesh names machines, + // which is a convention leaking into every module that implements a provision. + At string `json:"at,omitempty"` // Secret is the file on this machine holding that consumer's credential, sealed to it. // // Named rather than carried, for the same reason the private network's key is: the mesh @@ -331,7 +345,7 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant, continue } out[g.Provision] = append(out[g.Provision], Contribution{ - From: g.From, Node: g.Consumer, Values: g.Values, + From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Secret: grantPath(directories[g.Provision], g.Consumer), }) }