From ff912d0972167e4e72fd8ce788ad7e79776b29fe Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 31 Oct 2026 09:24:53 +0100 Subject: [PATCH] Serve route-proxy's routes from the seat mesh-provisioner's plans alone and drop its received routes file, since a route is a provision and the file is the deprecated path (issue 774, research 072 row 34; needs the mesh-sdk pin carrying plan entries' given and Conn.Read) --- internal/ownmanifest/public_issuer_test.go | 15 ++ modules/route-proxy/README.md | 33 +++-- .../cmd/route-provider/provider.go | 16 +- .../cmd/route-provider/provider_test.go | 4 +- modules/route-proxy/cmd/route-proxy/bus.go | 98 +++++++----- modules/route-proxy/cmd/route-proxy/main.go | 112 ++++---------- modules/route-proxy/cmd/route-proxy/plans.go | 123 +++++++++++++++ .../cmd/route-proxy/policy_test.go | 12 +- .../route-proxy/cmd/route-proxy/reach_test.go | 140 +++++++++++++----- .../cmd/route-proxy/routes_test.go | 69 ++++----- .../internal/routestate/routestate.go | 7 + modules/route-proxy/module.json | 11 -- 12 files changed, 411 insertions(+), 229 deletions(-) create mode 100644 modules/route-proxy/cmd/route-proxy/plans.go diff --git a/internal/ownmanifest/public_issuer_test.go b/internal/ownmanifest/public_issuer_test.go index b49e74d5..5cecb50c 100644 --- a/internal/ownmanifest/public_issuer_test.go +++ b/internal/ownmanifest/public_issuer_test.go @@ -1,6 +1,8 @@ package ownmanifest import ( + "path/filepath" + "slices" "testing" . "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -94,8 +96,21 @@ func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) { // 2 and 4): a Go bundle of the module's own Go module stamped for alpine, unpacked by an archive resource, mounted // read-only at /opt/mesh, called through args, and named under restart-on so a new program never leaves the old // one running. The container's name, network and logging are what the jail and the routes rely on. +// +// And its routes are the seat mesh-provisioner's plans alone (issue 774, research 072 row 34): it reads them, and +// receives no file of contributions; no module of this repository does. func TestRouteProxyRunsItsMountedProgramOnTheTrustImage(t *testing.T) { m := catalogueManifest(t, "route-proxy") + if !slices.Contains(m.Reads, "mesh-provisioner.plans") { + t.Errorf("route-proxy does not read mesh-provisioner.plans, where its routes are: %v", m.Reads) + } + dirs, _ := filepath.Glob(filepath.Join("..", "..", "modules", "*")) + for _, dir := range dirs { + if r := catalogueManifest(t, filepath.Base(dir)); len(r.Receives) > 0 { + t.Errorf("%s receives %v: a route, like every provision here, is served from the seat's plans, never a "+ + "received file", r.Module, r.Receives) + } + } var bundle *Artifact for i, a := range m.Build.Artifacts { if a.Name == "server" { diff --git a/modules/route-proxy/README.md b/modules/route-proxy/README.md index 2767eca7..ae9ca8d8 100644 --- a/modules/route-proxy/README.md +++ b/modules/route-proxy/README.md @@ -5,26 +5,31 @@ The shipping form of the reference reverse proxy (novox/hq ADR 0007, novox/hq 08 ## What it is -**A route is a grant.** A module that must be reachable declares `requires: ["route"]` and -contributes the name it wants and the port it listens on; this module provides `route`, receives -every consumer that asked as a file, and forwards by the `Host` header of each request to -`http://:` — where `at` is where the mesh says that consumer's machine is (empty means -this machine, reached over loopback). +**A route is a provision** (issue 774, research 072 row 34; the operator, 2026-10-31: "there is only 1 path, the other +is simply deprecated/invalid"). A module that must be reachable declares `requires: ["route"]` and contributes the +name it wants and the port it listens on; this module provides `route`. The controller composes each consumer's plan, +and the seat `mesh-provisioner` keeps it in its state `plans`: the plan's entry for the route names this proxy as +its provider and carries what the consumer contributes, composed (`given`: its names joined to the node's domains, its +endpoint's port, its policy), and where the consumer's machine is on the private network (`consumer-at`). The proxy +follows the plans on the bus and forwards by the `Host` header of each request to `http://:`, or +over loopback for a consumer on its own machine. -The control plane does not write the proxy's configuration. It delivers the facts — the -contributions file at `receives.route` — and the proxy turns them into whatever it runs. That -boundary is why swapping this proxy for another costs nothing to the modules that publish through -it. +The controller does not write the proxy's configuration. It composes the facts into the plans and the proxy turns them +into whatever it runs. That boundary is why swapping this proxy for another costs nothing to the modules that publish +through it. **There is no received file and no second source:** a route no plan names is not served, and a module +declaring `receives.route` is refused by the catalogue check (mesh-catalog `checks/credentialform`), but for the +migration scaffolding still listed there. | field | value | why | |---|---|---| -| `provides` | `route` (scope `mesh`) | a module on any node may require it | +| `provides` | `route` (scope `mesh`, answered by the helper) | a module on any node may require it | | `serves.route` | `{}` | a route hands back a name, not a credential — the consumer already knows the name it asked for | -| `receives.route` | `…/routes/mesh.json` | where the mesh writes every contribution; the proxy reads it as `$ROUTES` and re-reads on change | +| `reads` | `mesh-provisioner.plans`, `mesh-provisioner.provisions` | the plans are where the routes are; the helper reads both | | `listens` | `80` and `443`, both `from: anywhere` | the one machine with a public opening; the firewall opens these for free (ADR 0045) | -No provisioner: the proxy neither mints a credential for anybody nor answers a provision other -than `route`. It reads the file the mesh writes, and its own broker credential is for its tools. +Until the bus answers, the proxy routes nothing and says so on its log; it refuses to start without its broker +credential (`MESH_BROKER_FILE`). Who is inside the private network, and so may be served an internal-only name, is +still read from its membership (novox/hq ADR 0167). ## Which issuer: Let's Encrypt, stated here (novox/hq ADR 0226) @@ -77,7 +82,7 @@ traffic. | env | default | meaning | |---|---|---| -| `ROUTES` | `/routes/mesh.json` | the contributions file the mesh writes (`receives.route`) | +| `MESH_BROKER_FILE` | *(required)* | the proxy's broker credential: the plans it routes from, and its membership | | `LISTEN` | `:80` | HTTP, and the ACME HTTP-01 challenge | | `TLS_LISTEN` | `:443` | HTTPS; unset to serve plain HTTP only | | `ACME_CACHE` | `/acme` | where issued certificates persist; required when `TLS_LISTEN` is set, so a restart does not re-order | diff --git a/modules/route-proxy/cmd/route-provider/provider.go b/modules/route-proxy/cmd/route-provider/provider.go index 7eee671b..4e0a27cf 100644 --- a/modules/route-proxy/cmd/route-provider/provider.go +++ b/modules/route-proxy/cmd/route-provider/provider.go @@ -7,18 +7,20 @@ package main // **It composes nothing.** A route's public name is the label its consumer chose joined to the node's public domain, // which only the controller knows (the domain is placement), so the controller composes it and carries it in the // consumer's plan entry (`names`); the helper copies it into the entry's values unchanged. There is no credential for -// a route and nothing to make: the traefik files stay the TypeScript adapter's, written from the contributions the -// mesh still gives it. +// a route and nothing to make. The proxy itself serves the same plans' entries (cmd/route-proxy plans.go), the one +// source of routes (issue 774, research 072 row 34). import ( "git.novox.be/novox/mesh-sdk/go/provider" + + "route-proxy/internal/routestate" ) -// ProvisionName is what route-proxy provides, as its manifest says. -const ProvisionName = "route" - -// ModuleName is this module. -const ModuleName = "route-proxy" +// ProvisionName is what route-proxy provides, as its manifest says, and ModuleName this module. +const ( + ProvisionName = routestate.Provision + ModuleName = routestate.Module +) // newProvider is route-proxy answering its consumers over bus on node: the route, served as the manifest serves it // (nothing of its own), with no credential, and nothing to make or retire for a consumer. diff --git a/modules/route-proxy/cmd/route-provider/provider_test.go b/modules/route-proxy/cmd/route-provider/provider_test.go index 1c36b220..378832bd 100644 --- a/modules/route-proxy/cmd/route-provider/provider_test.go +++ b/modules/route-proxy/cmd/route-provider/provider_test.go @@ -108,6 +108,8 @@ func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { // the public ingress does not change. Everything of the manifest but its build, what it provides and the state it // keeps is pinned by this digest; changing the ingress means changing it here, on purpose. Re-pinned in R4d (novox/hq // ADR 0480): the program is an archive mounted into the trust image, the container and its files are otherwise as before. +// Re-pinned for issue 774 (research 072 row 34): the received routes file, its directory, mount and ROUTES went, since +// the proxy serves from the seat's plans alone; the ingress, its certificates and its files are otherwise as before. func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -133,7 +135,7 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { t.Fatal(err) } sum := sha256.Sum256(pinned) - if got := hex.EncodeToString(sum[:]); got != "85cb955e5a4b2d8826f50aa76433064a576e8b15b71537032eb93c7388c0ee28" { + if got := hex.EncodeToString(sum[:]); got != "e96702560003ae7f27490d27fd2353f8c6680d8e16864e15fdc0a833108bc2e5" { t.Fatalf("route-proxy's own manifest changed: digest %s", got) } // The two acme templates are pinned the same way (issue 849): they are what the proxy's certificates are asked diff --git a/modules/route-proxy/cmd/route-proxy/bus.go b/modules/route-proxy/cmd/route-proxy/bus.go index 9dbcc274..b8721838 100644 --- a/modules/route-proxy/cmd/route-proxy/bus.go +++ b/modules/route-proxy/cmd/route-proxy/bus.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "fmt" "log" "strings" @@ -10,9 +9,11 @@ import ( "github.com/nats-io/nats.go" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/provisions" "route-proxy/internal/routestate" ) @@ -27,28 +28,28 @@ func keptStates() []announce.State { } } -// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167). +// What the mesh says this proxy routes, and who is inside the mesh, read on the bus. // -// **The proxy is told, not left to work it out.** Its membership carries the routes it is given — -// the same contributions its file is written from — and every machine's address on the private -// network, which is who may be served an internal name. Read once at connect and followed, so a -// route added or a machine joining reaches a running proxy without a restart. +// **The routes are the seat mesh-provisioner's plans, alone** (issue 774, research 072 row 34): every plan's entry +// naming this proxy as a route's provider, followed, so a route added or withdrawn reaches a running proxy without a +// restart. **Who is inside** is this proxy's membership (novox/hq ADR 0160, ADR 0167): every machine's address on the +// private network, which is who may be served an internal name; until it is read, an internal name is served to this +// machine alone. -// followMembership connects with the credential in path and applies every membership the mesh -// issues this proxy. It retries the first connection for as long as it takes: a proxy that started -// before the bus keeps serving the file, and takes the bus when it answers. -func followMembership(path string, held *table, fromBus *atomic.Bool) { +// follow connects with the credential in path, follows the plans and the membership, and retries the connection for as +// long as it takes: until the bus answers, the proxy routes nothing, and says so. +func follow(path string, held *table) { for { - err := followOnce(path, held, fromBus) + err := followOnce(path, held) if err == nil { return } - log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err) + log.Printf("cannot follow the plans that say what this proxy routes, so it routes what it last read: %v", err) time.Sleep(30 * time.Second) } } -func followOnce(path string, held *table, fromBus *atomic.Bool) error { +func followOnce(path string, held *table) error { account, err := process.ReadAccount(path) if err != nil { return err @@ -65,42 +66,63 @@ func followOnce(path string, held *table, fromBus *atomic.Bool) error { } else { held.ledger.attach(account.Node, conn.State(routestate.RoutesState), conn.State(routestate.UnroutedState)) } - subject := membership.Subject(account.Node, account.Module) - err = conn.FollowMembership(func(issued membership.Membership) { - if took := applyMembership(issued, held); took && !fromBus.Swap(true) { - log.Printf("routes now come from this proxy's membership on %s", subject) - } - }, func(err error) { log.Print(err) }) - if err != nil { + if err := followPlans(conn.Read(provisions.PlansRead), newPlanned(account.Node), held); err != nil { conn.Close() return err } + subject := membership.Subject(account.Node, account.Module) + err = conn.FollowMembership(func(issued membership.Membership) { + applyMembership(issued, held) + }, func(err error) { log.Print(err) }) + if err != nil { + log.Printf("cannot follow this proxy's membership on %s, so an internal name is served to this machine "+ + "alone: %v", subject, err) + } return nil } -// applyMembership serves what a membership says, and says whether it said anything about routes. -// -// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays -// the source rather than every route being withdrawn because a field was absent. +// planWatch is the seat's plans as this proxy watches them: every current key, then every change. +type planWatch interface { + Watch(key string, handle func(stdio.StateChange) error) error +} + +// followPlans serves what the plans give this proxy: the whole table once every current plan is read, then again on +// every change that alters it. A table built from part of the plans would withdraw every route not yet read. +func followPlans(plans planWatch, given *planned, held *table) error { + var read atomic.Bool // every current plan is read: from here, each change is served as it comes + err := plans.Watch("", func(c stdio.StateChange) error { + var value []byte + if c.Op != "delete" { + value = c.Value + } + if changed := given.take(c.Key, value); changed && read.Load() { + serve(given, held) + } + return nil + }) + if err != nil { + return fmt.Errorf("watching %s: %w", provisions.PlansRead, err) + } + read.Store(true) + serve(given, held) + return nil +} + +// serve replaces what the proxy serves with what the plans give it. +func serve(given *planned, held *table) { + routes, public := routesOf(given.contributions()) + held.set(routes, public) + log.Printf("serving %d route(s) from the seat's plans: %s", len(routes), strings.Join(held.names(), ", ")) +} + +// applyMembership takes who is inside the mesh from a membership, and says whether it could read it. func applyMembership(issued membership.Membership, held *table) bool { - raw, carries := issued.Receives["route"] - if !carries { - return false - } - var contributions []contribution - if err := json.Unmarshal(raw, &contributions); err != nil { - log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err) - return false - } inside, err := sourcesOf(issued.Mesh) if err != nil { - log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err) + log.Printf("the mesh in this proxy's membership is unreadable, keeping who is inside as it was: %v", err) return false } - routes, public := routesOf(contributions) - held.set(routes, public) held.setInside(inside) - log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s", - len(routes), len(inside), strings.Join(held.names(), ", ")) + log.Printf("internal names are served to %d machine(s)", len(inside)) return true } diff --git a/modules/route-proxy/cmd/route-proxy/main.go b/modules/route-proxy/cmd/route-proxy/main.go index 5288f55f..905ce80c 100644 --- a/modules/route-proxy/cmd/route-proxy/main.go +++ b/modules/route-proxy/cmd/route-proxy/main.go @@ -1,13 +1,14 @@ // A reverse proxy, in the form the mesh expects one. // -// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable -// declares it requires `route` and contributes the name it wants; the proxy provides `route` and -// receives every consumer that asked. It is the mirror of a database grant: there the consumer +// **A route is a provision** (novox/hq ADR 0007, 08-connectivity §3; issue 774, research 072 row 34). A +// module that must be reachable declares it requires `route` and contributes the name it wants; the +// proxy provides `route` and serves every consumer whose plan, in the seat mesh-provisioner's state +// `plans`, names it as the route's provider. It is the mirror of a database grant: there the consumer // supplies a name and receives credentials, here it supplies a target and receives a name. // -// **It is an example, not part of the control plane.** The control plane decides and never touches -// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this -// program. What lives here is that contract, written as something that runs so it can be read +// **It is an example, not part of the controller.** The controller decides and never touches a +// machine. A real deployment runs whatever proxy it likes — the contract is the plan's entry, not +// this program. What lives here is that contract, written as something that runs so it can be read // rather than described. // // **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used @@ -16,11 +17,9 @@ // an open middleware surface recreates the thing being replaced and is far harder to narrow later // than a closed one is to widen. // -// What it is given, written by the host from an ordinary declaration: -// -// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is -// -// Each contribution's values carry the name and port as before, and optionally: +// What it is given, in each plan's entry for the route: every contribution the consumer makes, +// composed by the controller, and where the consumer's machine is on the private network (plans.go). +// Each contribution's values carry the name and port, and optionally: // // path the path prefix this rule is scoped to; absent means every path // priority which rule wins where two match; higher first, and the order is total @@ -35,8 +34,8 @@ // outright rather than served unprotected, and a secret that cannot be read makes the route refuse // rather than open — a gate that cannot check is not a gate that opens. // -// It re-reads on change rather than being restarted, for the same reason the provisioner does: -// a route arriving or leaving is an ordinary event and must not drop the connections of every +// It follows the plans on the bus rather than being restarted, for the same reason the provisioner +// does: a route arriving or leaving is an ordinary event and must not drop the connections of every // other workload. package main @@ -48,7 +47,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/hex" - "encoding/json" "fmt" "log" "net" @@ -61,7 +59,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" "golang.org/x/crypto/acme" @@ -119,11 +116,8 @@ func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy { } } -// what the mesh writes: the contributions file, one entry per consumer. -type given struct { - Given []contribution `json:"given"` -} - +// contribution is one route a consumer asked for: the module, its machine, where that machine is, and what it +// contributes, composed by the mesh (its plan's entry for the route, plans.go). type contribution struct { From string `json:"from"` Node string `json:"node"` @@ -183,9 +177,9 @@ type rule struct { from, node string } -// table is what the proxy is currently serving, replaced whole whenever the file changes. +// table is what the proxy is currently serving, replaced whole whenever the plans change what they give it. // -// Replaced rather than merged: the file is the whole truth about who has a route, so merging +// Replaced rather than merged: the plans are the whole truth about who has a route, so merging // would keep serving a name whose module was unassigned — which is the stale-route fault // 08-connectivity lists as open, reintroduced one level down. // @@ -356,7 +350,7 @@ func (t *table) routed(host string) bool { // // **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity // §2): a public name is certified by a public CA, an internal one by the mesh's own. This is -// composed only from `to` and `public`, which routesFrom already builds correctly — a host never +// composed only from `to` and `public`, which routesOf already builds correctly — a host never // lands in both a route's own `name` and only its `internal-name`, so nothing new has to be // tracked to tell the two apart. func (t *table) eligibleForInternalACME(host string) bool { @@ -432,50 +426,21 @@ func main() { } func run() error { - path := strings.TrimSpace(os.Getenv("ROUTES")) - if path == "" { - return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing") - } listen := strings.TrimSpace(os.Getenv("LISTEN")) if listen == "" { listen = ":80" } + // **The seat's plans are the one source of routes** (issue 774, research 072 row 34). Without the bus this proxy + // knows no route, so it refuses to start rather than serving an empty table and saying nothing. + credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")) + if credential == "" { + return fmt.Errorf("MESH_BROKER_FILE is not set, so this proxy cannot read the plans that say what it routes") + } held := newTable() held.ledger = newLedger() go held.ledger.loop() - // **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries - // the routes and who the mesh is; the file carries the routes alone, so while the proxy reads - // it an internal name is served to this machine and to nobody else — refused, never opened. - fromBus := &atomic.Bool{} - if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" { - go followMembership(credential, held, fromBus) - } else { - log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+ - "internal is served to this machine alone", path) - } - read := func() { - if fromBus.Load() { - return - } - routes, public, err := routesFrom(path) - if err != nil { - // Kept serving what it had. A file being rewritten is momentarily unreadable, and - // dropping every route because one read landed mid-write would turn an ordinary - // event into an outage. - log.Printf("cannot read %s, keeping what is already served: %v", path, err) - return - } - held.set(routes, public) - log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", ")) - } - read() - - go func() { - for range time.Tick(2 * time.Second) { - read() - } - }() + go follow(credential, held) // TLS is opt-in. A proxy with no `TLS_LISTEN` serves plain HTTP exactly as before — which is // what an internal-only mesh wants, and what the mesh's own certificate authority already @@ -881,30 +846,13 @@ func boolByte(b bool) byte { return 0 } -// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and -// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached -// only through `internal-name` never appears there. +// routesOf turns the routes the plans give into host → the rules for that host, and which of those hosts is a public +// name — the second is `name`, ACME-eligible; a host reached only through `internal-name` never appears there. // -// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches -// decides which names the mesh composes, so an endpoint that reaches only the private network -// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is -// served under its internal name and certified by the internal authority. Only a route with -// neither name has nothing to be served under (novox/hq issue 191). -func routesFrom(path string) (map[string][]rule, map[string]bool, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, nil, err - } - var said given - if err := json.Unmarshal(raw, &said); err != nil { - return nil, nil, err - } - routes, public := routesOf(said.Given) - return routes, public, nil -} - -// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public -// names — the same whether the contributions came in the file or in the membership. +// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches decides which names the +// mesh composes, so an endpoint that reaches only the private network arrives with an `internal-name` and no `name`. +// That is a whole route, not a malformed one: it is served under its internal name and certified by the internal +// authority. Only a route with neither name has nothing to be served under (novox/hq issue 191). func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) { out := map[string][]rule{} public := map[string]bool{} diff --git a/modules/route-proxy/cmd/route-proxy/plans.go b/modules/route-proxy/cmd/route-proxy/plans.go new file mode 100644 index 00000000..d42ea313 --- /dev/null +++ b/modules/route-proxy/cmd/route-proxy/plans.go @@ -0,0 +1,123 @@ +package main + +// The routes this proxy serves, read from the seat mesh-provisioner's plans and nothing else (issue 774; research 072 +// row 34, the operator on 2026-10-31: "there is only 1 path, the other is simply deprecated/invalid"). +// +// **A route is a provision.** A module that must be reachable requires `route`; the controller composes its plan on +// its node, and the plan's entry for the route names this proxy as its provider and carries what the module +// contributes, composed as the received file used to carry it (its names joined to the node's domains, its endpoint's +// port, its policy), and where the module's machine is on the private network. The proxy follows the plans on the bus +// and serves every entry naming it. There is no file and no second source: a route the plans do not name is not +// served. + +import ( + "fmt" + "log" + "sort" + "sync" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/provisions" + + "route-proxy/internal/routestate" +) + +// planned is the routes the seat's plans give this proxy on its node, by plan key, kept so a plan that cannot be read +// leaves what it gave served (a bind missed would read as a route withdrawn). +type planned struct { + node string + mu sync.Mutex + by map[string][]contribution +} + +func newPlanned(node string) *planned { + return &planned{node: node, by: map[string][]contribution{}} +} + +// take is one change of a plan's key: its value, or nil when the key was deleted. It answers whether what is served +// changed, and says on the log what it could not serve. +func (p *planned) take(key string, value []byte) bool { + p.mu.Lock() + defer p.mu.Unlock() + if value == nil { + _, had := p.by[key] + delete(p.by, key) + return had + } + gave, problems, err := routesOfPlan(p.node, key, value) + if err != nil { + log.Printf("the plan %s is not readable, so what it routes is kept as it was: %v", key, err) + return false + } + for _, problem := range problems { + log.Print(problem) + } + if len(gave) == 0 { + _, had := p.by[key] + delete(p.by, key) + return had + } + p.by[key] = gave + return true +} + +// contributions is every route the plans give this proxy, in the order of their plans' keys. +func (p *planned) contributions() []contribution { + p.mu.Lock() + defer p.mu.Unlock() + keys := make([]string, 0, len(p.by)) + for k := range p.by { + keys = append(keys, k) + } + sort.Strings(keys) + var out []contribution + for _, k := range keys { + out = append(out, p.by[k]...) + } + return out +} + +// routesOfPlan is the routes one plan gives the proxy on node: each contribution of each entry naming this proxy on +// node as the route's provider, reached on its own machine over loopback and on another at the address the plan says. +// A plan of another module, or one routing elsewhere, gives none. Problems are what the plan names and the proxy +// cannot serve, said rather than skipped silently: an entry carrying nothing to serve (a controller before the plans +// carried it), or a consumer elsewhere with no address on the private network. +func routesOfPlan(node, key string, value []byte) ([]contribution, []string, error) { + plan, err := provisions.DecodePlan(value) + if err != nil { + return nil, nil, err + } + if plan.Node+"."+plan.Module != key { + return nil, nil, fmt.Errorf("it is the plan of %s.%s, under the key %s", plan.Node, plan.Module, key) + } + var out []contribution + var problems []string + for _, e := range plan.Plan.Provisions { + if !routesHere(node, e) { + continue + } + if len(e.Given) == 0 { + problems = append(problems, fmt.Sprintf("the plan %s names this proxy for its route and gives nothing "+ + "to serve; it is not served", key)) + continue + } + at := "" + if plan.Node != node { + if e.ConsumerAt == "" { + problems = append(problems, fmt.Sprintf("the plan %s routes %s on %s here and says no address on "+ + "the private network for it; it is not served", key, plan.Module, plan.Node)) + continue + } + at = e.ConsumerAt + } + for _, g := range e.Given { + out = append(out, contribution{From: plan.Module, Node: plan.Node, At: at, Values: g.Values}) + } + } + return out, problems, nil +} + +// routesHere is whether a plan's entry is a route this proxy on node provides. +func routesHere(node string, e catalogue.PlanProvision) bool { + return e.Provision == routestate.Provision && e.Provider == node && e.ProviderModule == routestate.Module +} diff --git a/modules/route-proxy/cmd/route-proxy/policy_test.go b/modules/route-proxy/cmd/route-proxy/policy_test.go index 192af8d6..0f2b3646 100644 --- a/modules/route-proxy/cmd/route-proxy/policy_test.go +++ b/modules/route-proxy/cmd/route-proxy/policy_test.go @@ -57,11 +57,7 @@ func ask(t *testing.T, proxy, name, path string, auth [2]string) *http.Response func proxyFor(t *testing.T, routesJSON string) string { t.Helper() - path := filepath.Join(t.TempDir(), "routes.json") - if err := os.WriteFile(path, []byte(routesJSON), 0o644); err != nil { - t.Fatal(err) - } - routes, public, err := routesFrom(path) + routes, public, err := routesFrom(routesJSON) if err != nil { t.Fatal(err) } @@ -153,11 +149,7 @@ func TestACredentialInTheDeclarationIsRefusedRatherThanServed(t *testing.T) { `{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"$2a$10$abcdefghijklmnopqrstuv"}}]}`, } for _, body := range inline { - path := filepath.Join(t.TempDir(), "routes.json") - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - routes, _, err := routesFrom(path) + routes, _, err := routesFrom(body) if err != nil { t.Fatal(err) } diff --git a/modules/route-proxy/cmd/route-proxy/reach_test.go b/modules/route-proxy/cmd/route-proxy/reach_test.go index d008d649..5e70b2a9 100644 --- a/modules/route-proxy/cmd/route-proxy/reach_test.go +++ b/modules/route-proxy/cmd/route-proxy/reach_test.go @@ -9,9 +9,11 @@ import ( "net/http" "net/http/httptest" "net/url" + "reflect" "strings" "testing" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/membership" ) @@ -26,12 +28,12 @@ func behind(t *testing.T, mesh ...string) *table { at, _ := url.Parse(workload.URL) host, port, _ := net.SplitHostPort(at.Host) - routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[ + routes, public, err := routesFrom(fmt.Sprintf(`{"given":[ {"from":"app","node":"anchor","at":%q, "values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}}, {"from":"admin","node":"anchor","at":%q, "values":{"internal-name":"admin.anchor.internal","port":%s}} - ]}`, host, port, host, port))) + ]}`, host, port, host, port)) if err != nil { t.Fatal(err) } @@ -165,42 +167,114 @@ func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) { } } -// What the mesh issues is what is served: the routes in the membership, internal names to the -// machines it names (novox/hq ADR 0167). -func TestAMembershipIsServedAsIssued(t *testing.T) { +// fakePlans is the seat's plans as a watch hands them: every current key, then each change after Watch returned. +type fakePlans struct { + current map[string]string + handle func(stdio.StateChange) error +} + +func (f *fakePlans) Watch(_ string, handle func(stdio.StateChange) error) error { + f.handle = handle + for k, v := range f.current { + if err := handle(stdio.StateChange{Key: k, Op: "put", Value: json.RawMessage(v), Current: true}); err != nil { + return err + } + } + return nil +} + +// plan is a plan of the seat mesh-provisioner for module on node, its entries as given. +func plan(node, module, entries string) string { + return fmt.Sprintf(`{"v":1,"node":%q,"module":%q,"plan":{"templates":{"source":"","digest":""},"files":null,`+ + `"provisions":[%s]}}`, node, module, entries) +} + +// What the plans give is what is served, and nothing else (issue 774, research 072 row 34): each entry naming this +// proxy on its node as the route's provider, every contribution of it, reached over loopback on this machine and at +// the plan's address on another; an entry routing elsewhere, giving nothing or naming no address is not served. A +// change is served as it comes, a plan that cannot be read keeps what it gave, and a deleted one withdraws it. The +// membership says only who is inside: an internal-only name is served to the machines it names. +func TestThePlansAreServedAndTheMembershipSaysWhoIsInside(t *testing.T) { + const gitea = `{"provision":"route","provider":"anchor","provider-module":"route-proxy","consumer-at":"10.10.0.7",` + + `"given":[{"local":"web","values":{"name":"git.example","internal-name":"git.anchor.internal","port":3000}},` + + `{"local":"api","values":{"internal-name":"git-api.anchor.internal","port":3001}}]}` + plans := &fakePlans{current: map[string]string{ + "laptop.gitea": plan("laptop", "gitea", gitea+`,{"provision":"postgres-database","provider":"anchor",`+ + `"provider-module":"postgres","given":[{"values":{"name":"gitea"}}]}`), + "anchor.blog": plan("anchor", "blog", `{"provision":"route","provider":"anchor","provider-module":"route-proxy",`+ + `"consumer-at":"10.10.0.1","given":[{"values":{"name":"blog.example","port":2368}}]}`), + "anchor.other": plan("anchor", "other", `{"provision":"route","provider":"edge","provider-module":"route-proxy",`+ + `"given":[{"values":{"name":"other.example","port":80}}]}`), + "anchor.older": plan("anchor", "older", `{"provision":"route","provider":"anchor","provider-module":"route-proxy"}`), + "laptop.lost": plan("laptop", "lost", `{"provision":"route","provider":"anchor","provider-module":"route-proxy",`+ + `"given":[{"values":{"name":"lost.example","port":80}}]}`), + }} held := newTable() - took := applyMembership(membership.Membership{ - Receives: map[string]json.RawMessage{"route": json.RawMessage(`[ - {"from":"admin","node":"anchor","at":"anchor.internal", - "values":{"internal-name":"admin.anchor.internal","port":8080}}]`)}, - Mesh: []string{"10.10.0.7"}, - }, held) - if !took { - t.Fatal("a membership carrying routes was not applied") + if err := followPlans(plans, newPlanned("anchor"), held); err != nil { + t.Fatal(err) + } + served := func() map[string]string { + out := map[string]string{} + held.mu.RLock() + defer held.mu.RUnlock() + for host, rules := range held.to { + out[host] = rules[0].target + } + return out + } + want := map[string]string{"git.example": "http://10.10.0.7:3000", "git.anchor.internal": "http://10.10.0.7:3000", + "git-api.anchor.internal": "http://10.10.0.7:3001", "blog.example": "http://127.0.0.1:2368"} + if got := served(); !reflect.DeepEqual(got, want) { + t.Fatalf("serving %v, want %v", got, want) + } + if !held.eligibleForACME("git.example") || held.eligibleForACME("git-api.anchor.internal") { + t.Error("a public name is not certified publicly, or an internal-only one is") + } + for _, row := range []struct { + name, key, value string + served []string + withdrawn []string + }{ + {"a route arrives", "anchor.new", plan("anchor", "new", `{"provision":"route","provider":"anchor",`+ + `"provider-module":"route-proxy","given":[{"values":{"name":"new.example","port":81}}]}`), + []string{"new.example"}, nil}, + {"an unreadable plan keeps what it gave", "laptop.gitea", `{"v":1,`, []string{"git.example"}, nil}, + {"a deleted plan withdraws its routes", "laptop.gitea", "", nil, []string{"git.example", "git-api.anchor.internal"}}, + } { + change := stdio.StateChange{Key: row.key, Op: "put", Value: json.RawMessage(row.value)} + if row.value == "" { + change = stdio.StateChange{Key: row.key, Op: "delete"} + } + if err := plans.handle(change); err != nil { + t.Fatal(err) + } + got := served() + for _, host := range row.served { + if _, ok := got[host]; !ok { + t.Errorf("%s: %s is not served (%v)", row.name, host, got) + } + } + for _, host := range row.withdrawn { + if _, ok := got[host]; ok { + t.Errorf("%s: %s is still served", row.name, host) + } + } + } + + held = behind(t) + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code != http.StatusNotFound { + t.Errorf("before a membership, a machine of the mesh was served an internal-only name: %d", code) + } + if !applyMembership(membership.Membership{Mesh: []string{"10.10.0.7"}}, held) { + t.Fatal("a membership naming the mesh was not applied") } if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { t.Error("a machine the membership names was refused the internal-only route") } - if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound { - t.Errorf("a machine the membership does not name was served the internal-only route: %d", code) - } -} - -// A membership that says nothing about routes is one from a controller that does not issue them, -// and changes nothing: the file stays the source rather than every route being withdrawn. -func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) { - held := behind(t, "10.10.0.7") - before := held.names() - if applyMembership(membership.Membership{Mesh: []string{"10.10.0.7"}}, held) { - t.Error("a membership without routes was taken as the source of routes") - } - if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") { - t.Errorf("a membership without routes changed what is served: %v, was %v", got, before) - } - if applyMembership(membership.Membership{ - Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)}, - Mesh: []string{"not-an-address"}, - }, held) { + if applyMembership(membership.Membership{Mesh: []string{"not-an-address"}}, held) { t.Error("a membership whose mesh cannot be read was applied") } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { + t.Error("an unreadable membership changed who is inside") + } } diff --git a/modules/route-proxy/cmd/route-proxy/routes_test.go b/modules/route-proxy/cmd/route-proxy/routes_test.go index a8447f45..a29095e4 100644 --- a/modules/route-proxy/cmd/route-proxy/routes_test.go +++ b/modules/route-proxy/cmd/route-proxy/routes_test.go @@ -2,23 +2,26 @@ package main import ( "context" + "encoding/json" "fmt" "io" "net/http" "net/http/httptest" - "os" - "path/filepath" "strings" "testing" ) -func write(t *testing.T, body string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "routes.json") - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) +// routesFrom is the table a test's routes make, written as `{"given":[…]}`: the contributions the plans +// give the proxy (plans.go), each as routesOfPlan makes it. +func routesFrom(body string) (map[string][]rule, map[string]bool, error) { + var said struct { + Given []contribution `json:"given"` } - return path + if err := json.Unmarshal([]byte(body), &said); err != nil { + return nil, nil, err + } + routes, public := routesOf(said.Given) + return routes, public, nil } // plain is the table an ordinary set of routes makes: one host, one target, no policy. @@ -51,9 +54,9 @@ func targetOf(routes map[string][]rule, host string) string { // A route is a grant: the consumer supplies a target, and where that machine is comes from the // mesh rather than from a naming convention the proxy has to know. func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[ + routes, _, err := routesFrom(`{"contributions":1,"requirement":"route","given":[ {"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -68,10 +71,10 @@ func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) { // target — the same convenience a predecessor proxy gave for reaching a service over the VPN // without a public TLS round trip. func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -94,10 +97,10 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) { // one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing — // skipping it left every internal-only module unreachable by name (novox/hq issue 191). func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -123,9 +126,9 @@ func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) { // A route with no internal-name composed gets no second host — the ordinary case, unchanged. func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","values":{"name":"app.example","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -137,9 +140,9 @@ func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { // A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the // only thing that works when there is no private network. func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","values":{"name":"app.example","port":9000}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -151,10 +154,10 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { // A route may name a target reached over https, for a backend that terminates its own TLS — the // shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise. func TestARouteMayTargetHttps(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"mail","node":"anchor","at":"anchor.internal", "values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -319,10 +322,10 @@ func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) { // public CA can validate a private name, and asking anyway would only spend the account's rate // limit on an order that can never succeed. func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -341,10 +344,10 @@ func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) { // A certificate is asked of the *internal* authority only for a name that is routed here and is // not a route's own public name — the internal-network alias, never the route it accompanies. func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -385,10 +388,10 @@ func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) { // default refuses them long before the workload is reached. It is configuration beside `insecure`, // not a fifth policy — novox/hq ADR 0108 closed that set at four. func TestARouteMayLimitTheBodyItCarries(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"registry","node":"anchor","at":"anchor.internal", "values":{"name":"images.example","port":5000,"max-request-body":21474836480}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -403,9 +406,9 @@ func TestARouteMayLimitTheBodyItCarries(t *testing.T) { // Saying nothing leaves the route unlimited, which is what every route already got. func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal","values":{"name":"app.example","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -427,10 +430,10 @@ func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) { at := strings.TrimPrefix(workload.URL, "http://") host, port, _ := strings.Cut(at, ":") - routes, _, err := routesFrom(write(t, `{"given":[ - {"from":"registry","node":"anchor","at":"`+host+`", - "values":{"name":"images.example","port":`+port+`,"max-request-body":8}} - ]}`)) + routes, _, err := routesFrom(`{"given":[ + {"from":"registry","node":"anchor","at":"` + host + `", + "values":{"name":"images.example","port":` + port + `,"max-request-body":8}} + ]}`) if err != nil { t.Fatal(err) } @@ -491,7 +494,7 @@ func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) { {"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""}, {"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""}, } { - routes, public, err := routesFrom(write(t, `{"given":[`+row.given+`]}`)) + routes, public, err := routesFrom(`{"given":[` + row.given + `]}`) if err != nil { t.Fatal(err) } diff --git a/modules/route-proxy/internal/routestate/routestate.go b/modules/route-proxy/internal/routestate/routestate.go index 44dd82d8..fa7ac98e 100644 --- a/modules/route-proxy/internal/routestate/routestate.go +++ b/modules/route-proxy/internal/routestate/routestate.go @@ -14,6 +14,13 @@ import ( "time" ) +// Module is route-proxy, and Provision what it provides: what a plan's entry names when this proxy is a route's provider +// (both halves read the seat mesh-provisioner's plans for it, issue 774). +const ( + Module = "route-proxy" + Provision = "route" +) + // The states route-proxy keeps, by their local names (the buckets are route-proxy_). const ( // RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got. diff --git a/modules/route-proxy/module.json b/modules/route-proxy/module.json index 1a42825f..daddb91b 100644 --- a/modules/route-proxy/module.json +++ b/modules/route-proxy/module.json @@ -16,9 +16,6 @@ "serves": { "route": {} }, - "receives": { - "route": "${dir:routes-dir}/mesh.json" - }, "requires": [ "internal-acme-ca" ], @@ -73,12 +70,6 @@ "mode": "0700", "place": "mesh" }, - { - "id": "routes-dir", - "type": "directory", - "path": "/var/lib/route-proxy/routes", - "mode": "0700" - }, { "id": "acme-cache", "type": "directory", @@ -167,14 +158,12 @@ "${dir:state}/internal-acme.env" ], "volumes": [ - "${dir:routes-dir}:/routes:ro", "${dir:acme-cache}:/acme", "${dir:ca-dir}:/ca:ro", "${dir:mesh-state}/broker:/run/secrets/broker:ro", "${dir:state}/route-proxy:/opt/mesh:ro" ], "env": { - "ROUTES": "/routes/mesh.json", "LISTEN": ":80", "TLS_LISTEN": ":443", "ACME_CACHE": "/acme",