From 01d57b629f006688a293153559b7d0027201f9a5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 23 Sep 2026 23:19:12 +0200 Subject: [PATCH] A route says the largest body its proxy may carry, and both proxies honour it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The registry's public name is served by the predecessor with a twenty-gigabyte buffering middleware, because a registry takes image layers in single requests of gigabytes and a proxy's default turns every push into a 413 the registry never sees. A route contribution had no way to say so, so the mesh could not take the name over without losing what made it usable. The contribution now carries `max-request-body`, a whole positive number of bytes, and the catalogue holds every route to an agreed vocabulary — label or name, port, and the limit — refusing a key no proxy reads (a field that parses cleanly and does nothing is a promise nobody keeps) and a route with no port (unreachable by the proxy it just asked for, found at parse time rather than in a proxy's log). The mesh's own proxy reads the limit as written, refuses a body past it as 413 rather than the 502 the transport would have reported, and skips a route whose limit it cannot read rather than carrying what the module said not to. The registry's hand-over itself is read from the catalogue beside this checkout: the store still resolves with no proxy, the gate beside it pulls the store in, contributes the predecessor's name on the port the node gave it, and locks only the door that faces the world. hq ADR 0082/0104, the registry hand-over. --- examples/route-proxy/main.go | 123 +++++++++++++-- examples/route-proxy/routes_test.go | 110 +++++++++++-- internal/catalogue/manifest.go | 7 + internal/catalogue/registry_gate_test.go | 193 +++++++++++++++++++++++ internal/catalogue/route.go | 162 +++++++++++++++++++ internal/catalogue/route_test.go | 128 +++++++++++++++ 6 files changed, 695 insertions(+), 28 deletions(-) create mode 100644 internal/catalogue/registry_gate_test.go create mode 100644 internal/catalogue/route.go create mode 100644 internal/catalogue/route_test.go diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go index 2962a85..e580aac 100644 --- a/examples/route-proxy/main.go +++ b/examples/route-proxy/main.go @@ -12,7 +12,8 @@ // // What it is given, written by the host from an ordinary declaration: // -// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is +// $ROUTES every consumer, the name it asked for, where the mesh says that machine is, and +// the largest request body it will take (`max-request-body`, bytes; absent is no limit) // // It re-reads on change rather than being restarted, for the same reason the provisioner does: // a route arriving or leaving is an ordinary event and must not drop the connections of every @@ -27,8 +28,10 @@ import ( "crypto/x509" "encoding/hex" "encoding/json" + "errors" "fmt" "log" + "math" "net" "net/http" "net/http/httputil" @@ -95,6 +98,23 @@ type contribution struct { Values map[string]any `json:"values"` } +// route is one name the proxy serves: where it goes, and what it will not carry there. +type route struct { + // Target is the workload, as a URL. + Target string + // MaxRequestBody is the largest request body, in bytes, this route accepts — the + // contribution's `max-request-body`. Zero is no limit, which is what a route that said nothing + // gets: the mesh's own proxy has never limited a body, and a default that appeared with the + // field would have broken every route that did not ask for one. + MaxRequestBody int64 +} + +// served is a route the proxy has built: the reverse proxy, and the limit it enforces in front. +type served struct { + proxy *httputil.ReverseProxy + limit int64 +} + // table is what the proxy is currently serving, replaced whole whenever the file changes. // // Replaced rather than merged: the file is the whole truth about who has a route, so merging @@ -102,26 +122,28 @@ type contribution struct { // 08-connectivity lists as open, reintroduced one level down. type table struct { mu sync.RWMutex - to map[string]*httputil.ReverseProxy - targets map[string]string + to map[string]served + targets map[string]route } -func (t *table) set(routes map[string]string) { - made := map[string]*httputil.ReverseProxy{} - for name, target := range routes { - where, err := url.Parse(target) +func (t *table) set(routes map[string]route) { + made := map[string]served{} + for name, r := range routes { + where, err := url.Parse(r.Target) if err != nil { - log.Printf("route %s points at %q, which is not a URL: %v", name, target, err) + log.Printf("route %s points at %q, which is not a URL: %v", name, r.Target, err) continue } - made[name] = httputil.NewSingleHostReverseProxy(where) + proxy := httputil.NewSingleHostReverseProxy(where) + proxy.ErrorHandler = tooLargeOrBadGateway + made[name] = served{proxy: proxy, limit: r.MaxRequestBody} } t.mu.Lock() t.to, t.targets = made, routes t.mu.Unlock() } -func (t *table) find(host string) (*httputil.ReverseProxy, bool) { +func (t *table) find(host string) (served, bool) { // The port is not part of the name. A request to app.example:8080 is for app.example. if h, _, err := net.SplitHostPort(host); err == nil { host = h @@ -132,6 +154,24 @@ func (t *table) find(host string) (*httputil.ReverseProxy, bool) { return p, ok } +// tooLargeOrBadGateway is what the proxy answers when the workload could not be reached — or when +// it was the request that stopped, because its body ran past the route's limit. +// +// A body read that hits the limit surfaces as the transport's error, which the reverse proxy +// would report as 502 — the workload's fault, in the client's eyes, for a limit the client hit. +// Named as what it was: 413, so a push that is too large is told so rather than told the registry +// is down. +func tooLargeOrBadGateway(w http.ResponseWriter, r *http.Request, err error) { + var exceeded *http.MaxBytesError + if errors.As(err, &exceeded) { + http.Error(w, fmt.Sprintf("the request body for %q is larger than the %d bytes this route "+ + "accepts", r.Host, exceeded.Limit), http.StatusRequestEntityTooLarge) + return + } + log.Printf("http: proxy error: %v", err) + w.WriteHeader(http.StatusBadGateway) +} + func (t *table) names() []string { t.mu.RLock() defer t.mu.RUnlock() @@ -285,13 +325,13 @@ func forThisAuthority(cache, directory string, root []byte) string { // newTable is an empty routing table. func newTable() *table { - return &table{to: map[string]*httputil.ReverseProxy{}, targets: map[string]string{}} + return &table{to: map[string]served{}, targets: map[string]route{}} } // handler is the proxy itself, separated so it can be driven by a test without a listener. func handler(held *table) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - proxy, known := held.find(r.Host) + route, known := held.find(r.Host) if !known { // **Named, not a bare 404.** A route that was withdrawn and a name that never existed // are different things, and a proxy that says only "not found" makes an operator go @@ -302,12 +342,26 @@ func handler(held *table) http.Handler { r.Host, strings.Join(held.names(), ", ")) return } - proxy.ServeHTTP(w, r) + if route.limit > 0 { + // **The limit is the route's, enforced here rather than by the workload** — the + // contribution says what the proxy may carry to it, which is the one thing the + // workload cannot say for itself once a proxy stands in front. A body whose length is + // declared and too large is refused before a byte of it is read; one whose length is + // not declared is read up to the limit and refused at the byte past it. + if r.ContentLength > route.limit { + http.Error(w, fmt.Sprintf("the request body for %q is %d bytes, and this route "+ + "accepts at most %d", r.Host, r.ContentLength, route.limit), + http.StatusRequestEntityTooLarge) + return + } + r.Body = http.MaxBytesReader(w, r.Body, route.limit) + } + route.proxy.ServeHTTP(w, r) }) } -// routesFrom reads what the mesh wrote and turns it into name → target. -func routesFrom(path string) (map[string]string, error) { +// routesFrom reads what the mesh wrote and turns it into name → route. +func routesFrom(path string) (map[string]route, error) { raw, err := os.ReadFile(path) if err != nil { return nil, err @@ -317,7 +371,7 @@ func routesFrom(path string) (map[string]string, error) { return nil, err } - out := map[string]string{} + out := map[string]route{} for _, c := range said.Given { name, _ := c.Values["name"].(string) if name == "" { @@ -330,6 +384,16 @@ func routesFrom(path string) (map[string]string, error) { c.From, c.Node, name) continue } + // A limit it cannot honour is a route it does not serve — skipped and named, like a + // port that is not one. Serving the route with no limit instead would carry exactly what + // the module said not to carry, and report success. + limit, ok := bodyLimit(c.Values["max-request-body"]) + if !ok { + log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is not "+ + "a whole positive number of bytes; skipped", c.From, c.Node, name, + c.Values["max-request-body"]) + continue + } // Where the mesh says that machine is. Empty means it is this one — a workload beside the // proxy is ordinary, and reaching it over loopback is both correct and the only thing // that works when there is no private network. @@ -337,11 +401,36 @@ func routesFrom(path string) (map[string]string, error) { if at == "" { at = "127.0.0.1" } - out[strings.ToLower(name)] = fmt.Sprintf("http://%s:%d", at, port) + out[strings.ToLower(name)] = route{ + Target: fmt.Sprintf("http://%s:%d", at, port), + MaxRequestBody: limit, + } } return out, nil } +// bodyLimit reads a contribution's `max-request-body`: absent is no limit, and anything present +// must be a whole positive number of bytes — the same rule the control plane's catalogue applies +// when it parses the manifest, so a limit that reaches here has already passed it once. +func bodyLimit(v any) (int64, bool) { + if v == nil { + return 0, true + } + var n float64 + switch x := v.(type) { + case float64: + n = x + case int: + n = float64(x) + default: + return 0, false + } + if n < 1 || n != math.Trunc(n) || n > math.MaxInt64 { + return 0, false + } + return int64(n), true +} + // asPort accepts what JSON makes of a number, which is a float even when it was written 8080. func asPort(v any) (int, bool) { switch n := v.(type) { diff --git a/examples/route-proxy/routes_test.go b/examples/route-proxy/routes_test.go index 6740569..b5b8163 100644 --- a/examples/route-proxy/routes_test.go +++ b/examples/route-proxy/routes_test.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "context" + "io" "net/http" "net/http/httptest" "os" @@ -30,7 +32,7 @@ func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) { } // Lower-cased, because a Host header is not case-sensitive and a route that only answers the // spelling in the manifest answers half the requests made to it. - if routes["app.example"] != "http://laptop.internal:8080" { + if routes["app.example"].Target != "http://laptop.internal:8080" { t.Fatalf("the route does not point at the consumer: %v", routes) } } @@ -44,7 +46,7 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { if err != nil { t.Fatal(err) } - if routes["app.example"] != "http://127.0.0.1:9000" { + if routes["app.example"].Target != "http://127.0.0.1:9000" { t.Fatalf("a workload on this machine was not reachable: %v", routes) } } @@ -59,7 +61,7 @@ func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) { if err != nil { t.Fatal(err) } - if len(routes) != 1 || routes["fine.example"] == "" { + if len(routes) != 1 || routes["fine.example"].Target == "" { t.Fatalf("an unusable contribution was served: %v", routes) } } @@ -75,7 +77,7 @@ func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) { host, port, _ := strings.Cut(target, ":") held := newTable() - held.set(map[string]string{"app.example": "http://" + host + ":" + port}) + held.set(map[string]route{"app.example": {Target: "http://" + host + ":" + port}}) proxy := httptest.NewServer(handler(held)) defer proxy.Close() @@ -120,11 +122,11 @@ func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) { // nothing fails more visibly than a stale grant, which is exactly why it must not survive. func TestWithdrawingARouteStopsServingIt(t *testing.T) { held := newTable() - held.set(map[string]string{ - "going.example": "http://a.internal:80", - "staying.example": "http://b.internal:80", + held.set(map[string]route{ + "going.example": {Target: "http://a.internal:80"}, + "staying.example": {Target: "http://b.internal:80"}, }) - held.set(map[string]string{"staying.example": "http://b.internal:80"}) + held.set(map[string]route{"staying.example": {Target: "http://b.internal:80"}}) if _, still := held.find("going.example"); still { t.Fatal("a route whose module was unassigned is still served") @@ -137,7 +139,7 @@ func TestWithdrawingARouteStopsServingIt(t *testing.T) { // A Host header carries a port and the name does not. func TestARequestNamingAPortStillFindsItsRoute(t *testing.T) { held := newTable() - held.set(map[string]string{"app.example": "http://a.internal:8080"}) + held.set(map[string]route{"app.example": {Target: "http://a.internal:8080"}}) if _, found := held.find("app.example:8080"); !found { t.Fatal("a request to app.example:8080 did not find the route for app.example") } @@ -168,7 +170,7 @@ func TestTheIssuerIsStagingUnlessNamed(t *testing.T) { // rate limit — and the proxy would look healthy throughout. func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) { held := newTable() - held.set(map[string]string{"photos.example": "http://127.0.0.1:8080"}) + held.set(map[string]route{"photos.example": {Target: "http://127.0.0.1:8080"}}) policy := onlyWhatTheMeshSaid(held) if err := policy(context.Background(), "photos.example"); err != nil { @@ -184,7 +186,7 @@ func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) { // A route withdrawn stops being certifiable, without the proxy restarting. func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) { held := newTable() - held.set(map[string]string{"photos.example": "http://127.0.0.1:8080"}) + held.set(map[string]route{"photos.example": {Target: "http://127.0.0.1:8080"}}) policy := onlyWhatTheMeshSaid(held) if err := policy(context.Background(), "photos.example"); err != nil { t.Fatal(err) @@ -196,3 +198,89 @@ func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) { "once rather than what is served now") } } + +// The registry's hand-over (novox/hq ADR 0082, ADR 0104): a registry takes image layers in single +// requests of gigabytes, and its public name was served by the predecessor with a twenty-gigabyte +// body limit. The contribution now says so, and this proxy reads it as written — and a limit it +// cannot read is a route it does not serve, like a port that is not one. +func TestABodyLimitIsReadFromTheContributionOrTheRouteIsSkipped(t *testing.T) { + routes, err := routesFrom(write(t, `{"given":[ + {"from":"gate","node":"anchor","values":{"name":"registry-api.example","port":5001,"max-request-body":21474836480}}, + {"from":"app","node":"anchor","values":{"name":"app.example","port":8080}}, + {"from":"odd","node":"anchor","values":{"name":"odd.example","port":8081,"max-request-body":"20g"}}, + {"from":"none","node":"anchor","values":{"name":"none.example","port":8082,"max-request-body":0}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if got := routes["registry-api.example"].MaxRequestBody; got != 21474836480 { + t.Errorf("the limit did not arrive as written: %d", got) + } + if got := routes["app.example"].MaxRequestBody; got != 0 { + t.Errorf("a route that asked for no limit was given one: %d", got) + } + for _, skipped := range []string{"odd.example", "none.example"} { + if _, served := routes[skipped]; served { + t.Errorf("%s asked for a limit that is not a number of bytes and was served anyway", skipped) + } + } +} + +// A body past the route's limit is refused as too large — whether its length is declared up front +// or only discovered while it is read — and a body within it reaches the workload whole. Refused +// as 413, not 502: a push that is too large must be told so, not told the registry is down. +func TestABodyPastTheRoutesLimitIsRefusedAsTooLarge(t *testing.T) { + var received int64 + workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n, _ := io.Copy(io.Discard, r.Body) + received = n + w.WriteHeader(http.StatusCreated) + })) + defer workload.Close() + + held := newTable() + held.set(map[string]route{ + "limited.example": {Target: workload.URL, MaxRequestBody: 1024}, + "unlimited.example": {Target: workload.URL}, + }) + proxy := httptest.NewServer(handler(held)) + defer proxy.Close() + + push := func(host string, body []byte, declared bool) int { + t.Helper() + var reader io.Reader = bytes.NewReader(body) + if !declared { + // A reader that is not a bytes.Reader carries no length: the request goes out chunked + // and the proxy learns the size only by reading it. + reader = io.MultiReader(bytes.NewReader(body)) + } + asked, err := http.NewRequest(http.MethodPut, proxy.URL+"/v2/blob", reader) + if err != nil { + t.Fatal(err) + } + asked.Host = host + answer, err := http.DefaultClient.Do(asked) + if err != nil { + t.Fatal(err) + } + defer answer.Body.Close() + _, _ = io.Copy(io.Discard, answer.Body) + return answer.StatusCode + } + + small, large := bytes.Repeat([]byte("x"), 1000), bytes.Repeat([]byte("y"), 4096) + + if got := push("limited.example", small, true); got != http.StatusCreated || received != 1000 { + t.Fatalf("a body within the limit got %d and %d bytes arrived", got, received) + } + if got := push("limited.example", large, true); got != http.StatusRequestEntityTooLarge { + t.Fatalf("a declared body past the limit got %d, not 413", got) + } + if got := push("limited.example", large, false); got != http.StatusRequestEntityTooLarge { + t.Fatalf("an undeclared body past the limit got %d, not 413", got) + } + // And a route that asked for no limit carries whatever it is given. + if got := push("unlimited.example", large, true); got != http.StatusCreated || received != 4096 { + t.Fatalf("a route with no limit refused or truncated a body: %d, %d bytes", got, received) + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index e39fa17..eae5002 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -870,6 +870,13 @@ func ParseManifest(raw []byte) (Manifest, error) { // round, and both are better said plainly. problems = append(problems, fmt.Sprintf( "%s contributes nothing to %q; if it only needs one, require it", m.Module, to)) + continue + } + if to == RouteProvision { + // The one provision whose contribution has an agreed vocabulary (route.go): every + // proxy reads the same keys, so a key none of them reads is refused here rather than + // carried to a proxy that ignores it. + problems = append(problems, routeProblems(m.Module, values)...) } } problems = append(problems, m.Build.problems(m.Module)...) diff --git a/internal/catalogue/registry_gate_test.go b/internal/catalogue/registry_gate_test.go new file mode 100644 index 0000000..3268c7d --- /dev/null +++ b/internal/catalogue/registry_gate_test.go @@ -0,0 +1,193 @@ +package catalogue + +import ( + "encoding/json" + "strings" + "testing" +) + +// The registry's public name, taken over from the predecessor (novox/hq ADR 0082, ADR 0104, the +// registry hand-over) — read from the catalogue beside this checkout, parsed by the real parser. +// +// **The public door is a second module beside the store, not a route on the store.** Contributing +// a route is requiring one, and the store is raised at genesis on a node with no proxy; a store +// that required a route would be a store no first node could have. So `distribution` stays the +// registry ADR 0082 describes — reached by name, over the private network, with no account — and +// `distribution-gate` is a second registry process on the same volume, behind the registry's own +// basic auth, with the public name. The mesh's own pulls never pass through it, which is provable +// from the two manifests: the store's container carries no auth and mounts no htpasswd. + +// aStubProxy provides `route` so a declaration can be made without a built artifact: the real +// providers are the adapter (whose container is a mesh-built artifact) and the proxy. +func aStubProxy() Manifest { + return Manifest{Module: "proxy", Version: "1", + Provides: FromAnywhere("route"), + Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}} +} + +func TestTheStoreNeedsNoRouteAndTheGateBringsTheStoreBesideIt(t *testing.T) { + store := catalogueManifest(t, "distribution") + gate := catalogueManifest(t, "distribution-gate") + adapter := catalogueManifest(t, "route-adapter") + + // The store alone, with nothing providing a route — genesis' own set — still resolves. + alone, err := Resolve(shelf(store, gate, adapter), []string{"distribution"}, workstation(), World{}) + if err != nil { + t.Fatalf("the store cannot be resolved without a proxy, so no first node could have one: %v", err) + } + if got := names(alone); len(got) != 1 || got[0] != "distribution" { + t.Fatalf("the store alone resolved to %v", got) + } + + // The gate wants the store's storage, which is a node-scoped provision: assigning the gate + // brings the store in beside it — the two share a volume, and a gate on a machine without the + // store would serve an empty one. And `route` resolves from the adapter exactly as from the + // proxy (ADR 0104). + together, err := Resolve(shelf(store, gate, adapter), + []string{"distribution-gate", "route-adapter"}, withDomain("example.test"), World{}) + if err != nil { + t.Fatalf("the gate does not resolve beside the adapter: %v", err) + } + for _, want := range []string{"distribution", "distribution-gate", "route-adapter"} { + if !among(names(together), want) { + t.Errorf("%s is missing from %v", want, names(together)) + } + } + if because := together.Because["distribution"]; !strings.Contains(because, "distribution-gate") { + t.Errorf("the store was not pulled in by the gate: %q", because) + } +} + +func TestTheGateContributesTheRegistrysPublicNameAndAGivenPortFollowsIntoIt(t *testing.T) { + store := catalogueManifest(t, "distribution") + gate := catalogueManifest(t, "distribution-gate") + + got, err := Resolve(shelf(store, gate, aStubProxy()), + []string{"distribution-gate", "proxy"}, withDomain("example.test"), World{}) + if err != nil { + t.Fatal(err) + } + // The node gave the gate's port a machine port (novox/hq ADR 0100) — on the machine being + // migrated the predecessor's interface still holds the default — as the operator does, with the + // `ports` setting. + moved, err := GivenPorts(gate, []Layer{{From: "node anchor", + Values: map[string]any{PortsSetting: map[string]any{"5001": float64(5101)}}}}) + if err != nil { + t.Fatalf("the gate's port cannot be given a machine port: %v", err) + } + out, err := got.Declaration(Rendering{ + Ports: map[string]map[int]int{"distribution-gate": moved}, + Given: map[string]map[int]int{"distribution-gate": moved}, + Needed: map[string]map[string]string{ + "distribution": {"broker": "sealed-broker"}, + "distribution-gate": {"htpasswd": "sealed"}, + }, + }) + if err != nil { + t.Fatal(err) + } + + var given []Contribution + var storeContainer, gateContainer, storeConfig, gateConfig, htpasswd map[string]any + for _, r := range out { + switch { + case r["path"] == "/var/lib/proxy/routes.json": + var parsed struct { + Given []Contribution `json:"given"` + } + if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil { + t.Fatal(err) + } + given = parsed.Given + case r["name"] == "mesh-registry": + storeContainer = r + case r["name"] == "mesh-registry-gate": + gateContainer = r + case r["path"] == "/var/lib/mesh/registry/config.yml": + storeConfig = r + case r["path"] == "/var/lib/mesh/registry-gate/config.yml": + gateConfig = r + case r["path"] == "/var/lib/mesh/registry-gate/htpasswd": + htpasswd = r + } + } + + // One route: the predecessor's name, composed from the label and the node's domain, on the + // port the machine actually published, with the limit a layer push needs. + if len(given) != 1 || given[0].From != "distribution-gate" { + t.Fatalf("the proxy was given %v", given) + } + v := given[0].Values + if v["name"] != "registry-api.example.test" { + t.Errorf("the public name did not compose: %v", v["name"]) + } + if port, _ := asPort(v["port"]); port != 5101 { + t.Errorf("the route points at %v, and the machine published the gate on 5101", v["port"]) + } + if limit, ok := RouteBodyLimit(v["max-request-body"]); !ok || limit != 21474836480 { + t.Errorf("the route carries a body limit of %v; a layer push needs the predecessor's twenty gigabytes", v["max-request-body"]) + } + + // The lock is the registry's own, and only on the door that faces the world: the gate mounts + // the operator's htpasswd and its configuration names it; the store does neither, so what the + // mesh pulls over the private network needs no account (ADR 0082). + if htpasswd == nil || htpasswd["sealed"] != "sealed" { + t.Fatalf("the gate's htpasswd is not delivered as a sealed own secret: %v", htpasswd) + } + if !mounts(gateContainer, "/var/lib/mesh/registry-gate/htpasswd:/etc/docker/registry/htpasswd:ro") { + t.Errorf("the gate does not mount the htpasswd: %v", gateContainer["volumes"]) + } + if !strings.Contains(gateConfig["content"].(string), "auth:\n htpasswd:") { + t.Errorf("the gate's configuration does not lock the door: %s", gateConfig["content"]) + } + if strings.Contains(storeConfig["content"].(string), "auth:") { + t.Errorf("the store's configuration asks for an account, and the mesh has none to give (ADR 0082): %s", storeConfig["content"]) + } + for _, v := range storeContainer["volumes"].([]any) { + if strings.Contains(v.(string), "htpasswd") { + t.Errorf("the store mounts an htpasswd: %v", v) + } + } + if env, has := storeContainer["env"]; has { + t.Errorf("the store's container carries an environment it did not before: %v", env) + } + + // Two processes, one store: both containers mount the same volume, and neither keeps a + // per-process descriptor cache over it. + for _, c := range []map[string]any{storeContainer, gateContainer} { + if !mounts(c, "mesh-registry-data:/var/lib/registry") { + t.Errorf("%v does not mount the registry's volume: %v", c["name"], c["volumes"]) + } + } + for _, cfg := range []map[string]any{storeConfig, gateConfig} { + if strings.Contains(cfg["content"].(string), "blobdescriptor") { + t.Errorf("%v keeps a per-process blob cache over a store two processes write: %s", cfg["path"], cfg["content"]) + } + if !strings.Contains(cfg["content"].(string), "delete:\n enabled: true") { + t.Errorf("%v lost the predecessor's delete setting, which tag retention depends on", cfg["path"]) + } + } + // And the machine published the gate where the node said. + if ports, _ := gateContainer["ports"].([]any); len(ports) != 1 || ports[0] != "5101:5001" { + t.Errorf("the gate is published as %v, and the node gave its 5001 the machine port 5101", gateContainer["ports"]) + } +} + +func mounts(container map[string]any, volume string) bool { + listed, _ := container["volumes"].([]any) + for _, v := range listed { + if v == volume { + return true + } + } + return false +} + +func among(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} diff --git a/internal/catalogue/route.go b/internal/catalogue/route.go new file mode 100644 index 0000000..3dc0cdc --- /dev/null +++ b/internal/catalogue/route.go @@ -0,0 +1,162 @@ +package catalogue + +import ( + "fmt" + "math" + "sort" + "strings" +) + +// What a module may say when it contributes a route. +// +// **The contract is the file, and this is its vocabulary** (novox/hq ADR 0007, 08-connectivity §3). +// A module reachable by name requires `route` and contributes what the proxy has to know; the +// mesh's own proxy (`examples/route-proxy`) and the adapter to a predecessor's (the catalogue's +// `route-adapter`, ADR 0104) both read the same contribution, which is what lets one stand in for +// the other without a module that publishes through them noticing. So the keys are agreed here, +// once, and a manifest is refused for a key no proxy reads: a field that parses cleanly and does +// nothing is a promise nobody keeps, and the module goes on believing its route is limited when it +// is not. +// +// The control plane still does not know what a reverse proxy *is* — it validates the shape and +// carries the values, and turning them into a router, a middleware or a body limit is the +// provider's. What is checked is exactly what every provider needs to be true: a name to serve, a +// port to send to, and a limit that is a number of bytes. + +// RouteProvision is the provision a module reachable by name requires. +const RouteProvision = "route" + +// RouteLabel is the subdomain a module asks to be published under; the node's public domain is +// joined to it (ADR 0066, composeName). +const RouteLabel = "label" + +// RouteName is the legacy full name, left untouched when a module still writes one. +const RouteName = "name" + +// RoutePort is the port the contributing workload's software uses. The mesh redirects it to +// wherever the machine published it (ADR 0038, atMachinePort) before the proxy sees it. +const RoutePort = "port" + +// RouteMaxRequestBody is the largest request body, in bytes, the proxy may accept for this route. +// +// **The registry's hand-over is why it exists** (novox/hq ADR 0082, the registry hand-over). A +// registry takes image layers in single requests of gigabytes, and a proxy's default limit — a +// megabyte, in some — turns every push into a 413 that the registry never sees. The predecessor +// served the registry's public name with exactly this limit as a middleware; a route that could not +// say it would have a public name it could not be pushed to. Absent, the proxy applies whatever it +// does by default, which for the mesh's own is no limit at all. +const RouteMaxRequestBody = "max-request-body" + +// routeKeys is every key a route contribution may carry, in the order a refusal names them. +var routeKeys = []string{RouteLabel, RouteName, RoutePort, RouteMaxRequestBody} + +// routeProblems is everything wrong with one module's route contribution, empty when nothing is. +// +// Read from the manifest as written: a per-node setting laid over it (settle) is a fact about one +// machine and is checked where settings are; this is the module's own promise. +func routeProblems(module string, values map[string]any) []string { + var problems []string + known := map[string]bool{} + for _, k := range routeKeys { + known[k] = true + } + var unknown []string + for k := range values { + if !known[k] { + unknown = append(unknown, k) + } + } + sort.Strings(unknown) + if len(unknown) > 0 { + problems = append(problems, fmt.Sprintf( + "%s contributes %s to %q, which no proxy reads — a route carries %s", + module, quoted(unknown), RouteProvision, strings.Join(routeKeys, ", "))) + } + + label, hasLabel := values[RouteLabel] + name, hasName := values[RouteName] + if !hasLabel && !hasName { + problems = append(problems, fmt.Sprintf( + "%s contributes a route and names nothing — a %q is the subdomain the node's public "+ + "domain is joined to", module, RouteLabel)) + } + if hasLabel && !aText(label) { + problems = append(problems, fmt.Sprintf( + "%s contributes a route whose %q is %v, and a label is a non-empty string", + module, RouteLabel, label)) + } + if hasName && !aText(name) { + problems = append(problems, fmt.Sprintf( + "%s contributes a route whose %q is %v, and a name is a non-empty string", + module, RouteName, name)) + } + + port, hasPort := values[RoutePort] + if !hasPort { + // Refused here rather than skipped by the proxy: a module that asked for a route and not + // for the port is unreachable by the proxy it just asked for (08-connectivity §3), and the + // proxy saying so in a log is the fault found at the wrong end. + problems = append(problems, fmt.Sprintf( + "%s contributes a route and no %q — the proxy has nowhere to send it", module, RoutePort)) + } else if n, ok := asPort(port); !ok || float64(n) != asNumber(port) || n < 1 || n > 65535 { + problems = append(problems, fmt.Sprintf( + "%s contributes a route on port %v, which is not a port", module, port)) + } + + if limit, said := values[RouteMaxRequestBody]; said { + if _, ok := RouteBodyLimit(limit); !ok { + problems = append(problems, fmt.Sprintf( + "%s contributes a route whose %q is %v, and a limit is a whole number of bytes, "+ + "at least one", module, RouteMaxRequestBody, limit)) + } + } + return problems +} + +// RouteBodyLimit reads a contribution's request-body limit: a whole, positive number of bytes. +// +// Shared with nothing that runs on a machine — the proxies read the file with their own parsers — +// but the rule they apply is this one, and a test holds each of them to it. +func RouteBodyLimit(v any) (int64, bool) { + var n float64 + switch x := v.(type) { + case float64: + n = x + case int: + n = float64(x) + case int64: + n = float64(x) + default: + return 0, false + } + if n < 1 || n != math.Trunc(n) || n > math.MaxInt64 { + return 0, false + } + return int64(n), true +} + +// aText is a non-empty string. +func aText(v any) bool { + s, ok := v.(string) + return ok && strings.TrimSpace(s) != "" +} + +// asNumber is a number's value whatever JSON or a test made of it, NaN otherwise. +func asNumber(v any) float64 { + switch n := v.(type) { + case float64: + return n + case int: + return float64(n) + } + return math.NaN() +} + +// quoted is a list as a refusal names it. +func quoted(keys []string) string { + out := make([]string, len(keys)) + for i, k := range keys { + out[i] = fmt.Sprintf("%q", k) + } + return strings.Join(out, ", ") +} diff --git a/internal/catalogue/route_test.go b/internal/catalogue/route_test.go new file mode 100644 index 0000000..3efe64b --- /dev/null +++ b/internal/catalogue/route_test.go @@ -0,0 +1,128 @@ +package catalogue + +import ( + "encoding/json" + "strings" + "testing" +) + +// A route contribution has an agreed vocabulary (route.go), and the parser holds a manifest to it: +// a key no proxy reads is refused rather than carried, a route with no port is refused rather than +// skipped by the proxy it asked for, and a body limit is a whole number of bytes or nothing. +// +// The limit is here for the registry's hand-over (novox/hq ADR 0082, ADR 0104): a registry takes +// image layers in single requests of gigabytes, and the predecessor served its public name with a +// twenty-gigabyte middleware. A route that could not say so would have a name it could not be +// pushed to. + +func routed(contribution string) ([]byte, error) { + raw := []byte(`{"module":"app","version":"1","contributes":{"route":` + contribution + `}}`) + _, err := ParseManifest(raw) + return raw, err +} + +func TestARouteWithALabelAndAPortIsAccepted(t *testing.T) { + if _, err := routed(`{"label":"git","port":3000}`); err != nil { + t.Fatalf("the shape every routed module in the catalogue writes was refused: %v", err) + } + // The legacy shape — a full name and no label — still passes, so the catalogue can migrate + // module by module (ADR 0066). + if _, err := routed(`{"name":"git.example","port":3000}`); err != nil { + t.Fatalf("a legacy full-name contribution was refused: %v", err) + } + // And a limit on what may be pushed through it. + if _, err := routed(`{"label":"registry-api","port":5001,"max-request-body":21474836480}`); err != nil { + t.Fatalf("a route with a body limit was refused: %v", err) + } +} + +func TestARouteKeyNoProxyReadsIsRefusedByName(t *testing.T) { + _, err := routed(`{"label":"git","port":3000,"basic-auth":true,"timeout":30}`) + if err == nil { + t.Fatal("a route carrying keys no proxy reads was accepted; the module goes on believing " + + "its route is limited when it is not") + } + for _, want := range []string{`"basic-auth"`, `"timeout"`, "max-request-body"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not name %s, leaving the author guessing: %v", want, err) + } + } +} + +func TestARouteWithNoPortIsRefused(t *testing.T) { + // A module that asked for a route and not for the port is unreachable by the proxy it just + // asked for (08-connectivity §3) — found at parse time, not in a proxy's log. + if _, err := routed(`{"label":"git"}`); err == nil || !strings.Contains(err.Error(), "port") { + t.Fatalf("a route with nowhere to send it was accepted: %v", err) + } + for _, bad := range []string{`"3000"`, `0`, `70000`, `3000.5`, `true`} { + if _, err := routed(`{"label":"git","port":` + bad + `}`); err == nil { + t.Errorf("a route on port %s was accepted, and that is not a port", bad) + } + } + // And a route that names nothing. + if _, err := routed(`{"port":3000}`); err == nil || !strings.Contains(err.Error(), "label") { + t.Fatalf("a route naming nothing was accepted: %v", err) + } + if _, err := routed(`{"label":"","port":3000}`); err == nil { + t.Fatal("a route with an empty label was accepted") + } +} + +func TestABodyLimitIsAWholePositiveNumberOfBytes(t *testing.T) { + for _, bad := range []string{`"20g"`, `"21474836480"`, `0`, `-1`, `1.5`, `true`, `null`} { + _, err := routed(`{"label":"registry-api","port":5001,"max-request-body":` + bad + `}`) + if err == nil || !strings.Contains(err.Error(), "max-request-body") { + t.Errorf("a body limit of %s was accepted, or refused without naming the key: %v", bad, err) + } + } + for _, v := range []any{float64(1), float64(21474836480), 1024, int64(4096)} { + if _, ok := RouteBodyLimit(v); !ok { + t.Errorf("%v (%T) is a whole positive number of bytes and was refused", v, v) + } + } + for _, v := range []any{"1", float64(0), float64(0.5), nil, true} { + if n, ok := RouteBodyLimit(v); ok { + t.Errorf("%v (%T) was read as a limit of %d bytes", v, v, n) + } + } +} + +// The limit reaches the proxy exactly as written, beside the composed name and the port — the +// mesh carries it and interprets nothing. +func TestABodyLimitReachesTheProxyUnchanged(t *testing.T) { + registry := Manifest{Module: "gate", Version: "1", + Contributes: map[string]map[string]any{ + "route": {"label": "registry-api", "port": 5001, "max-request-body": float64(21474836480)}, + }} + proxy := Manifest{Module: "proxy", Version: "1", + Provides: Offers("route"), + Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}} + got, err := Resolve(shelf(proxy, registry), []string{"gate"}, withDomain("example.test"), World{}) + if err != nil { + t.Fatal(err) + } + for _, r := range mustDeclare(t, got) { + if r["path"] != "/var/lib/proxy/routes.json" { + continue + } + var parsed struct { + Given []Contribution `json:"given"` + } + if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil { + t.Fatal(err) + } + if len(parsed.Given) != 1 { + t.Fatalf("the proxy was given %d routes", len(parsed.Given)) + } + v := parsed.Given[0].Values + if v["name"] != "registry-api.example.test" { + t.Errorf("the name did not compose: %v", v) + } + if limit, ok := RouteBodyLimit(v["max-request-body"]); !ok || limit != 21474836480 { + t.Errorf("the body limit did not reach the proxy as written: %v", v["max-request-body"]) + } + return + } + t.Fatal("the proxy was given no file") +}