novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Remove route-proxy from the controller's repository, which it left for mesh-networking

Browse Source 
The group's check composed route-proxy twice, here and in mesh-networking; the move is one delivery group
(issue 1126), so it leaves here in the same group. The tests that read its directory go with it.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 18:14:41 +01:00
parent 5c281ad99b
commit 4964ad1037
31 changed files with 0 additions and 5343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/gosource/gosource_test.go
-35
View File
@@ -172,41 +172,6 @@ func TestANarrowedTreeHoldsTheBuildSourceAndNothingElse(t *testing.T) {
}
}
// This repository's own programs: the route proxy lives in its module's own Go module, so its build source is
// not the controller's, and neither holds the other's command.
func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) {
root := filepath.Join("..", "..")
proxy, err := GoBuildSource(filepath.Join(root, "modules", "route-proxy"), "cmd/route-proxy")
if err != nil {
t.Fatal(err)
}
controller, err := GoBuildSource(root, "cmd/mesh-controller")
if err != nil {
t.Fatal(err)
}
for _, c := range []struct {
entries []string
name string
file string
held bool
}{
{proxy, "the route proxy", "cmd/route-proxy/main.go", true},
{proxy, "the route proxy", "cmd/route-proxy/bus.go", true},
{proxy, "the route proxy", "go.mod", true},
{proxy, "the route proxy", "cmd/route-provider/main.go", false},
{proxy, "the route proxy", "README.md", false},
{controller, "the controller", "cmd/mesh-controller/main.go", true},
{controller, "the controller", "internal/conditions/condition.go", true},
{controller, "the controller", "internal/inventory/migrations/0001-nodes.sql", true},
{controller, "the controller", "modules/route-proxy/cmd/route-proxy/main.go", false},
{controller, "the controller", "README.md", false},
} {
if SourceHolds(c.entries, c.file) != c.held {
t.Errorf("%s: %s held %v, wanted %v", c.name, c.file, !c.held, c.held)
}
}
}
// C or assembly beside Go holds its package's directory whole — an include may name a file below it — and a
// go.mod made between the module's root and a package is a change.
func TestNativeSourcesHoldTheirDirectoryWhole(t *testing.T) {
internal/ownmanifest/image_place_test.go
-22
View File
@@ -43,28 +43,6 @@ func imagesOutOfPlace(t *testing.T, root string, manifests map[string][]byte) []
return out
}
// Every image a module of this repository declares is built from a Dockerfile in the module's own directory
// (novox/hq ADR 0487 clause 13). The catalogue's own modules are held to the same by checks/imageplace in mesh-catalog.
func TestEveryImageArtifactBuildsFromADockerfileInItsOwnModuleDirectory(t *testing.T) {
root := filepath.Join("..", "..", "modules")
paths, err := filepath.Glob(filepath.Join(root, "*", "module.json"))
if err != nil || len(paths) == 0 {
t.Fatalf("no module manifests found in this repository's modules: %v", err)
}
manifests := map[string][]byte{}
for _, p := range paths {
raw, err := os.ReadFile(p)
if err != nil {
t.Fatal(err)
}
manifests[filepath.Base(filepath.Dir(p))] = raw
}
if got := imagesOutOfPlace(t, root, manifests); len(got) != 0 {
t.Errorf("these image artifacts do not build from a Dockerfile in their own module's directory "+
"(ADR 0487 clause 13): %v", got)
}
}
// The check fails on each bad case and passes on a good one, so it holds something.
func TestAnImageWhoseDockerfileIsOutOfPlaceIsFoundAndOneInPlaceIsNot(t *testing.T) {
root := t.TempDir()
internal/ownmanifest/public_issuer_test.go
-160
View File
@@ -1,160 +0,0 @@
package ownmanifest
import (
"path/filepath"
"slices"
"testing"
. "git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The public issuer is the proxy's own fact (novox/hq ADR 0226), and the folding of it must not
// move the proxy's account.
//
// route-proxy keeps each ACME authority's account and certificates in a directory named after a
// digest of the directory URL and of the root bundle its `trust` container copies in
// (modules/route-proxy/cmd/route-proxy, forThisAuthority). Until ADR 0226 the URL was rendered from a binding to
// `public-acme`'s `acme-ca`, spelled with the port. A URL spelled any other way — even the same
// authority without `:443` — or a root bundle from another image is a new authority to the proxy:
// a new account, and every routed name ordered again, on two machines at once, against Let's
// Encrypt's rate limits. So what the module now states is held to exactly what the binding rendered.
// renderedBeforeTheFold is route-proxy's acme.env as the binding to public-acme rendered it on every
// machine running the proxy, read from the controller's plan on 2026-10-06.
const renderedBeforeTheFold = "ACME_DIRECTORY=https://acme-v02.api.letsencrypt.org:443/directory\n" +
"ACME_ROOTS=https://acme-v02.api.letsencrypt.org:443\n" +
"ACME_ROOTS_PATH=\n"
// trustImage is the image whose system bundle becomes the public root the account directory is
// named after. Moving it renames that directory.
const trustImage = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) {
m := catalogueManifest(t, "route-proxy")
for _, r := range m.Requires {
if r == "acme-ca" {
t.Fatal("route-proxy still asks for acme-ca; the public issuer is its own fact (ADR 0226)")
}
}
// As a build would leave it: each artifact a container names resolved to an image. Which image
// does not matter to the file checked here. An archive resource carries no image: it unpacks a bundle.
for _, res := range m.Resources {
if res["type"] == "archive" {
continue
}
if artifact, ok := res["artifact"].(string); ok {
delete(res, "artifact")
res["image"] = "registry.invalid/route-proxy/" + artifact +
"@sha256:1111111111111111111111111111111111111111111111111111111111111111"
}
}
r := Resolution{
Node: "anchor",
Modules: []Manifest{m},
Needs: []Needed{{
Name: "internal-acme-ca", From: "home", At: "home.internal", For: "route-proxy",
Serves: map[string]any{
"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
},
}},
}
// Its own broker credential, sealed as the mesh would; what it is does not matter here.
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{
"route-proxy": {"broker": "sealed"}}})
if err != nil {
t.Fatalf("route-proxy could not be composed for a machine: %v", err)
}
env := fileNamed(out, "route-proxy.acme-env")
if env == nil {
t.Fatalf("nothing writes the public issuer's environment: %v", out)
}
if got, _ := env["content"].(string); got != renderedBeforeTheFold {
t.Fatalf("acme.env changed, which moves the proxy's account and reorders every certificate:\n"+
"got %q\nwant %q", got, renderedBeforeTheFold)
}
if fileNamed(out, "route-proxy.bound-acme-ca") != nil {
t.Error("a binding to acme-ca is still written")
}
var trust string
if m.Build != nil {
for _, a := range m.Build.Artifacts {
if a.Name == "trust" {
trust = a.From
}
}
}
if trust != trustImage {
t.Errorf("the trust image is %q, not %q: its system bundle is the public root the account "+
"directory is named after, so moving it reorders every certificate. Move it only with a "+
"plan for the account (ADR 0226)", trust, trustImage)
}
}
// The proxy's program is mounted into the pinned trust image, not baked into an image (novox/hq ADR 0480 clauses 1,
// 2 and 4): a Go bundle of the module's own Go module stamped for alpine, unpacked by an archive resource, mounted
// read-only at /opt/mesh, called through args, and named under restart-on so a new program never leaves the old
// one running. The container's name, network and logging are what the jail and the routes rely on.
//
// And its routes are the seat mesh-provisioner's plans alone (issue 774, research 072 row 34): it reads them, and
// receives no file of contributions; no module of this repository does.
func TestRouteProxyRunsItsMountedProgramOnTheTrustImage(t *testing.T) {
m := catalogueManifest(t, "route-proxy")
if !slices.Contains(m.Reads, "mesh-provisioner.plans") {
t.Errorf("route-proxy does not read mesh-provisioner.plans, where its routes are: %v", m.Reads)
}
dirs, _ := filepath.Glob(filepath.Join("..", "..", "modules", "*"))
for _, dir := range dirs {
if r := catalogueManifest(t, filepath.Base(dir)); len(r.Receives) > 0 {
t.Errorf("%s receives %v: a route, like every provision here, is served from the seat's plans, never a "+
"received file", r.Module, r.Receives)
}
}
var bundle *Artifact
for i, a := range m.Build.Artifacts {
if a.Name == "server" {
bundle = &m.Build.Artifacts[i]
}
if a.Kind == "image" {
t.Errorf("artifact %s is an image built from a Dockerfile (ADR 0452 §1)", a.Name)
}
}
if bundle == nil || bundle.Kind != "bundle" || bundle.Language != "go" || bundle.System != "alpine" ||
bundle.From != "cmd/route-proxy" || bundle.Binary != "route-proxy" {
t.Fatalf("the server artifact is not a Go bundle of cmd/route-proxy stamped for alpine: %+v", bundle)
}
var archive, server map[string]any
for _, r := range m.Resources {
switch {
case r["type"] == "archive" && r["artifact"] == "server":
archive = r
case r["id"] == "server":
server = r
}
}
if archive == nil || server == nil {
t.Fatalf("archive %v or server container %v is missing", archive, server)
}
if server["artifact"] != "trust" || server["name"] != "route-proxy" || server["network"] != "host" || server["logging"] != "journald" {
t.Errorf("the server container moved: %v", server)
}
mount := archive["path"].(string) + ":/opt/mesh:ro"
mounted := false
for _, v := range server["volumes"].([]any) {
mounted = mounted || v == mount
}
if !mounted {
t.Errorf("the archive is not mounted read-only at /opt/mesh (%s): %v", mount, server["volumes"])
}
if args, _ := server["args"].([]any); len(args) != 1 || args[0] != "/opt/mesh/route-proxy" {
t.Errorf("args are %v, want the mounted program", server["args"])
}
restarts := false
for _, v := range server["restart-on"].([]any) {
restarts = restarts || v == archive["id"]
}
if !restarts {
t.Errorf("restart-on %v does not name the archive %v", server["restart-on"], archive["id"])
}
}
modules/route-proxy/README.md
-130
View File
@@ -1,130 +0,0 @@
# route-proxy — the mesh's public ingress
The shipping form of the reference reverse proxy (novox/hq ADR 0007, novox/hq 08-connectivity §3
"Exposure"). It is the `traefik` replacement: the module that **provides `route`**.
## What it is
**A route is a provision** (issue 774, research 072 row 34; the operator, 2026-10-31: "there is only 1 path, the other
is simply deprecated/invalid"). A module that must be reachable declares `requires: ["route"]` and contributes the
name it wants and the port it listens on; this module provides `route`. The controller composes each consumer's plan,
and the seat `mesh-provisioner` keeps it in its state `plans`: the plan's entry for the route names this proxy as
its provider and carries what the consumer contributes, composed (its names joined to the node's domains, its
endpoint's port, its policy), and where the consumer's machine is on the private network, as one opaque payload
(`contributed`) the seat never reads. The proxy follows the plans on the bus, reads each contribution with its own
contract (below), and forwards by the `Host` header of each request to `http://<at>:<port>`, or over loopback for a
consumer on its own machine.
## Its contract: a package of its own
**route-proxy defines its own contract, and other modules use it** (the operator, 2026-10-31). It is the Go module
`git.novox.be/novox/mesh-controller/modules/route-proxy/contract`, in `contract/` here with its own `go.mod`, standard
library only, released by tags `modules/route-proxy/contract/vX.Y.Z`, the way mesh-sdk is released (`go/vX.Y.Z`); this
module depends on it by its version like any other. It holds:
- **the route a consumer asks** (`Request`, read strictly by `DecodeRequest`: no field it does not name, a name of
either kind, a port where the rule forwards, http or https, a positive body limit, a secret's file never a
credential) and its JSON schema (`route.schema.json`, `Schema`), the form of a manifest's `contributes.route`;
- **the events this proxy says** on `mesh.mod.route-proxy.event.<type>`: `route.served` and `route.withdrawn` as the
names it serves change, `route.refused` with why for a route a plan gives it and it does not serve (`Event`);
- **what it keeps on the bus**, its states `routes` and `unrouted`, their keys and records (`Route`, `Unrouted`,
`Other`, `Key`, `Split`);
- **its tools' names and arguments** (`ToolRoutes`, `ToolRoute`, `ToolUnrouted`, `RoutesArgs`, `RouteArgs`).
The controller does not write the proxy's configuration. It composes the facts into the plans and the proxy turns them
into whatever it runs. That boundary is why swapping this proxy for another costs nothing to the modules that publish
through it. **There is no received file and no second source:** a route no plan names is not served, and a module
declaring `receives.route` is refused by the catalogue check (mesh-catalog `checks/credentialform`), but for the
migration scaffolding still listed there.
| field | value | why |
|---|---|---|
| `provides` | `route` (scope `mesh`, answered by the helper) | a module on any node may require it |
| `serves.route` | `{}` | a route hands back a name, not a credential — the consumer already knows the name it asked for |
| `reads` | `mesh-provisioner.plans`, `mesh-provisioner.provisions`, `mesh-delivery-assigner.placement` | the plans are where the routes are; the helper reads both; the placement says who is inside |
| `emits` | `route.served`, `route.withdrawn`, `route.refused` | what it serves, said as it changes (the contract's `Event`) |
| `listens` | `80` and `443`, both `from: anywhere` | the one machine with a public opening; the firewall opens these for free (ADR 0045) |
Until the bus answers, the proxy routes nothing and says so on its log; it refuses to start without its broker
credential (`MESH_BROKER_FILE`). Who is inside the private network, and so may be served an internal-only name, is
every node's address on it as the seat mesh-placement decides it (mesh-sdk `delivery.InsideAddresses`); until that
is read, an internal-only name is served to this machine alone.
## Which issuer: Let's Encrypt, stated here (novox/hq ADR 0226)
**The public issuer is this module's own fact, not a provision.** Until 2026-10-06 it came from a
second module, `public-acme`, that ran nothing and offered `acme-ca` pointing at Let's Encrypt; every
node running a proxy was assigned it too, and nothing else ever consumed it. It is retired: the proxy
names the production directory in its own `acme.env`, and the only issuer it is still bound to is the
mesh's own (`internal-acme-ca`, from `step-ca`).
**`acme.env` is byte for byte what the binding rendered.** The proxy keeps each authority's account
and certificates in a directory under `/var/lib/route-proxy/acme` named after a digest of the
directory URL — `https://acme-v02.api.letsencrypt.org:443/directory`, port included, as the binding
spelled it — and of the root bundle the `trust` container copies in. Any change to either is a new
authority to the proxy: a new account, and every routed name ordered again against Let's Encrypt's
rate limits. So the URL keeps the `:443`, `ACME_ROOTS_PATH` stays empty, and the `trust` artifact's
pinned image is not moved without a decision; mesh-controller's
`TestRouteProxyKeepsItsPublicAccountDirectory` holds all three.
## How it ships the Go proxy
The proxy is a Go program, unlike the TypeScript tool-runtime modules. Its source sits in this module, at
`cmd/route-proxy`, in the module's own Go module (`go.mod`, with a `vendor/` directory), the contract written as
something that runs. No image is built for it (novox/hq ADR 0480): the `server` artifact is a Go bundle compiled
for alpine, an `archive` resource unpacks it onto the machine, and the `route-proxy` container runs the pinned
`trust` image (the same alpine digest whose system roots the public account directory is named after) with the
archive mounted read-only at `/opt/mesh` and `/opt/mesh/route-proxy` as its program. A new build of the program
restarts the container, because the archive is in its `restart-on`.
In the lab, a scenario stocks the same alpine image and serves it by digest.
## Certificates, and the endpoint (resolves 04-ISSUES/004)
TLS is opt-in and terminates here for public names, via ACME (autocert). A proxy with no
`TLS_LISTEN` serves plain HTTP; with it, it obtains a publicly-trusted certificate for exactly the
names the mesh routes here (`HostPolicy` = only routed names, so a scan cannot spend the account's
issuance quota) and answers the HTTP-01 challenge on `:80` at the name being certified.
**`ACME_DIRECTORY` defaults to Let's Encrypt *staging*, not production.** This closes
novox/hq 04-ISSUES/004 (certificate issuance targets production):
the issuer is a per-node choice, and the safe endpoint is the default. Production issuance is
rate-limited per domain and per account and does not replenish quickly; defaulting to production
would leave the safe path depending on somebody remembering to opt out of it, on exactly the work
most likely to iterate. A staging certificate is trusted by no browser, so the mistake announces
itself on the first request rather than a fortnight later at the rate limit.
**The module states production.** The binary's staging default stands for anything that runs it
without the module — the lab, a developer's machine — and the module's `acme.env` names production
for every node it is assigned to, because a node assigned the public proxy is one that serves public
traffic.
| env | default | meaning |
|---|---|---|
| `MESH_BROKER_FILE` | *(required)* | the proxy's broker credential: the plans it routes from, and its membership |
| `LISTEN` | `:80` | HTTP, and the ACME HTTP-01 challenge |
| `TLS_LISTEN` | `:443` | HTTPS; unset to serve plain HTTP only |
| `ACME_CACHE` | `/acme` | where issued certificates persist; required when `TLS_LISTEN` is set, so a restart does not re-order |
| `ACME_DIRECTORY` | Let's Encrypt **staging** in the binary; production in the module's `acme.env` | the issuer |
| `ACME_CA_BUNDLE` | *(unset)* | a file of roots to trust for the issuer's own API — set only for a private/lab authority whose API certificate the world does not yet trust |
## Answered by the provider helper (issue 792)
A small Go program beside the proxy (`cmd/route-provider`) runs the SDK's provider helper for `route`, with no credential,
and the offer says `"answered-by": "helper"`: a handed consumer (gitea's origins file) is filled by its machine's client
from this module's entry, which carries the public names the controller composed for it. The module is **not** marked
moved: its own files, `acme.env` and the broker secret, are composed as before, and the proxy itself is unchanged.
## What it says on the bus, and its tools (issue 774)
The proxy keeps two states, announced readable (`conn.Keeps`, novox/hq ADR 0495), each keyed `<node>.<name>`:
| state | one key per | holds | written when |
|---|---|---|---|
| `routes` | served name | the rules it is served by (path, module, node, target, refusal, redirect, authentication); its certificate (authority, issuer, not-after, last renewal, outcome, or why it failed); the last upstream answer (status, time, error, path) | a route arrives, changes or leaves; an authority stores a certificate; a handshake fails to find one or hands out one not yet recorded; an upstream answer differs from the last one written, or the same one is a minute old |
| `unrouted` | name answered "no route for", in a request or a TLS handshake, under a domain the proxy serves | when it was last asked, how often since the start, whether it is an internal-only name asked from outside; never who asked (the journal keeps that, for fail2ban) | first asked, at most five new names a minute, then at most once a minute; deleted when the name is served; kept a week. Every other refusal is only counted, in one key per proxy (`<node>._other`: the last minute's count and the total), written once a minute. A refused write is dropped, never retried |
Nothing is written on a timer. The provider helper's binary serves the module's tools from those states, one call each:
`route_proxy_routes` (every served name with its target, certificate expiry and last answer), `route_proxy_route`
(one name in full, or that it is not served and when it was last asked for) and `route_proxy_unrouted` (the names
refused, newest first).
modules/route-proxy/cmd/route-provider/main.go
-45
View File
@@ -1,45 +0,0 @@
// route-provider: route-proxy's Go half, one binary the node's runtime launches and speaks MCP to over stdio
// through the Go SDK. It runs the SDK's provider helper that answers the consumers of `route` (issue 792), and serves
// the module's tools, which read what the proxy keeps on the bus (issue 774, tools.go).
//
// stdout is the MCP channel, so everything this module says, it says on stderr.
package main
import (
"context"
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
func say(format string, args ...any) {
fmt.Fprintf(os.Stderr, "[route-proxy] "+format+"\n", args...)
}
func main() {
node := os.Getenv("MESH_NODE")
if node == "" {
say("MESH_NODE is not set; answering no consumers")
} else {
p := newProvider(node, provider.StateBus(ModuleName), say)
go func() {
// The helper's watches go through the runtime's channel, which Serve holds: one asked before it does is
// refused ("before Serve"), Run returns and the helper stays stopped until the process restarts.
<-stdio.Served()
if err := p.Run(context.Background()); err != nil {
say("the provider helper stopped: %v", err)
}
}()
}
tools := front{routes: stdio.State(contract.RoutesState), unrouted: stdio.State(contract.UnroutedState),
now: time.Now}.tools()
if err := stdio.Serve("", tools); err != nil {
say("%v", err)
os.Exit(1)
}
}
modules/route-proxy/cmd/route-provider/provider.go
-38
View File
@@ -1,38 +0,0 @@
package main
// route-proxy handed to the provisioner (novox/hq ADR 0349 clause 12, issue 792): the module runs the SDK's provider
// helper for `route`, so a consumer of a route — a handed gitea reading `${bound:route:name-web}` — is told its public
// name from the entry in this module's bucket, the one its machine's client fills its files from.
//
// **It composes nothing.** A route's public name is the label its consumer chose joined to the node's public domain,
// which only the controller knows (the domain is placement), so the controller composes it and carries it in the
// consumer's plan entry (`names`); the helper copies it into the entry's values unchanged. There is no credential for
// a route and nothing to make. The proxy itself serves the same plans' entries (cmd/route-proxy plans.go), the one
// source of routes (issue 774, research 072 row 34).
import (
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// ProvisionName is what route-proxy provides, as its manifest says, and ModuleName this module.
const (
ProvisionName = contract.Provision
ModuleName = contract.Module
)
// newProvider is route-proxy answering its consumers over bus on node: the route, served as the manifest serves it
// (nothing of its own), with no credential, and nothing to make or retire for a consumer.
func newProvider(node string, bus provider.Bus, logf func(string, ...any)) *provider.Provider {
return provider.New(provider.Options{
Node: node,
Module: ModuleName,
Offers: map[string]provider.Offer{ProvisionName: {
Credential: provider.NoCredential,
Serves: map[string]any{},
}},
Bus: bus,
Logf: logf,
})
}
modules/route-proxy/cmd/route-provider/provider_test.go
-159
View File
@@ -1,159 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// bucketBus is the helper's bus over in-memory buckets.
type bucketBus struct {
mu sync.Mutex
buckets map[string]map[string][]byte
}
func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error {
b.mu.Lock()
defer b.mu.Unlock()
for k, v := range b.buckets[bucket] {
handle(k, v)
}
return nil
}
func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error {
b.mu.Lock()
defer b.mu.Unlock()
if b.buckets[bucket] == nil {
b.buckets[bucket] = map[string][]byte{}
}
b.buckets[bucket][key] = append([]byte(nil), value...)
return nil
}
func (b *bucketBus) Delete(_ context.Context, bucket, key string) error {
b.mu.Lock()
defer b.mu.Unlock()
delete(b.buckets[bucket], key)
return nil
}
// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792),
// from the seat mesh-provisioner's plans and keys (novox/hq ADR 0570 clause 2), never the controller's assignments: a
// module not handed whose route the seat keys (grafana, issue 774) has a plan of provisions alone, and a helper that
// read only the assignments left its key asked for good, so the assigner withheld every new build of 38 such modules.
func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) {
bus := &bucketBus{buckets: map[string]map[string][]byte{}}
for module, names := range map[string]map[string]string{
"gitea": {"name": "git.example.org", "internal-name": "git.novox.internal", "name-web": "git.example.org"},
"grafana": {"name": "grafana.example.org"},
} {
plan, err := provisions.EncodePlan(provisions.Plan{Node: "g14", Module: module, Plan: catalogue.Plan{
Provisions: []catalogue.PlanProvision{{Provision: ProvisionName, Provider: "novox", ProviderModule: ModuleName,
At: "10.7.0.1", From: "novox", As: "mesh_g14_" + module, Names: names}}}})
if err != nil {
t.Fatal(err)
}
key := "g14." + module
if err := bus.Put(context.Background(), provisions.PlansBucket, key, plan); err != nil {
t.Fatal(err)
}
asked, err := json.Marshal(provisions.Provision{V: provisions.Version, Provision: ProvisionName,
Consumer: provisions.End{Node: "g14", Module: module}, Provider: provisions.End{Node: "novox", Module: ModuleName},
State: provisions.Asked})
if err != nil {
t.Fatal(err)
}
if err := bus.Put(context.Background(), provisions.ProvisionsBucket, key+"."+ProvisionName, asked); err != nil {
t.Fatal(err)
}
}
p := newProvider("novox", bus, t.Logf)
out, err := p.Pass(context.Background())
if err != nil {
t.Fatal(err)
}
if len(out) != 2 || out[0].Status != provider.Answered || out[1].Status != provider.Answered {
t.Fatalf("the pass did %+v", out)
}
for key, want := range map[string]map[string]string{
"g14.gitea.route": {"name-web": "git.example.org", "name": "git.example.org", "internal-name": "git.novox.internal"},
"g14.grafana.route": {"name": "grafana.example.org"},
} {
var e provider.Entry
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)][key], &e); err != nil {
t.Fatalf("%s: %v", key, err)
}
for k, v := range want {
if e.Values[k] != v {
t.Errorf("%s says %s %v, want %s", key, k, e.Values[k], v)
}
}
if e.State != provisions.Provided || e.VaultName != "" || e.User != "" || e.Refused != "" {
t.Errorf("the entry %s is %+v", key, e)
}
}
}
// route-proxy's own files and resources stay as they were (issue 792): the module answers its route through the
// helper but is not marked moved, so its acme.env, its broker secret and its containers are composed as today and
// the public ingress does not change. Everything of the manifest but its build, what it provides and the state it
// keeps is pinned by this digest; changing the ingress means changing it here, on purpose. Re-pinned in R4d (novox/hq
// ADR 0480): the program is an archive mounted into the trust image, the container and its files are otherwise as before.
// Re-pinned for issue 774 (research 072 row 34): the received routes file, its directory, mount and ROUTES went, since
// the proxy serves from the seat's plans alone; the ingress, its certificates and its files are otherwise as before.
func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
if _, moved := m["moved"]; moved {
t.Fatal("route-proxy is marked moved: its own files would be handed")
}
provides, _ := m["provides"].([]any)
if len(provides) != 1 || provides[0].(map[string]any)["answered-by"] != "helper" {
t.Fatalf("route is not said to be answered by the helper: %v", provides)
}
// Its states and its tools are what it says on the bus, not what it runs or writes (issue 774).
for _, k := range []string{"build", "provides", "state", "reads", "tools", "replaces"} {
delete(m, k)
}
// Refreshed on purpose for the events its contract names (`emits`: route.served, route.withdrawn, route.refused).
pinned, err := json.Marshal(m) // sorted keys, compact
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(pinned)
if got := hex.EncodeToString(sum[:]); got != "3a44c56cc021be4008f521d16fa02e1c0f0d2eedab2f4d0b60f7d3177269e542" {
t.Fatalf("route-proxy's own manifest changed: digest %s", got)
}
// The two acme templates are pinned the same way (issue 849): they are what the proxy's certificates are asked
// with, so a change to either is a change to the public ingress.
for file, want := range map[string]string{
"acme.env": "d66eba145c893080e30e013329fc00e51395f0c31fc8621a20c7b26abc11b52f",
"internal-acme.env": "baf4da75379183758c8ae505be4dd8a79d5b089ae1e7c8ded4d232df40251df5",
} {
text, err := os.ReadFile("../../files/" + file)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(text)
if got := hex.EncodeToString(sum[:]); got != want {
t.Fatalf("route-proxy's files/%s changed: digest %s, pinned %s; if the change is meant, check that the "+
"public and internal ingress still ask for the certificates they should, then pin the new digest here",
file, got, want)
}
}
}
modules/route-proxy/cmd/route-provider/tools.go
-296
View File
@@ -1,296 +0,0 @@
package main
// route-proxy's tools (issue 774): the web front's command line. Each answers from what the proxy keeps on the bus
// (route-proxy_routes and route-proxy_unrouted, written on change by the proxy itself), in one call, for every
// machine running a proxy: which sites are served and where they go, whether their certificates are good, what the
// last request through each got back, and which names were asked for and are served nowhere.
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// The tools' names, as the manifest declares them.
const (
RoutesTool = contract.ToolRoutes
RouteTool = contract.ToolRoute
UnroutedTool = contract.ToolUnrouted
)
// reader is one of the module's states, as stdio.KeptState reads it.
type reader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// front is what the tools read: the routes and the refusals, and the clock a certificate's days left are counted by.
type front struct {
routes, unrouted reader
now func() time.Time
}
// RouteLine is one served name in the list: where it goes, how long its certificate holds, what it last answered.
type RouteLine struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
Reach string `json:"reach"`
Targets []string `json:"targets"`
// Reachable is yes when the last answer upstream gave was below 500, no when it was 500 or more or none came,
// unknown before any request.
Reachable string `json:"reachable"`
// Certificate is its outcome, issuer and expiry in words; ExpiresInDays is the whole days until not-after.
Certificate string `json:"certificate"`
ExpiresInDays *int `json:"expires_in_days,omitempty"`
LastAnswer *contract.Answer `json:"last_answer,omitempty"`
}
func (f front) tools() []stdio.Tool {
node := map[string]any{"type": "string", "description": "only the proxy on this machine; every machine when left out"}
return []stdio.Tool{
{Name: RoutesTool,
Description: "Every site (web name, host name) the mesh's web front serves through route-proxy, the reverse proxy " +
"on the public machine: for each route the module and node it forwards to, its TLS certificate's issuer and " +
"expiry (days left), and the last answer upstream gave (status, time, error), so is X reachable and is its " +
"certificate good is answered without curl. Read from the proxy's routes state on the bus. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
Run: func(args map[string]any) (any, error) {
at, _ := args["node"].(string)
return f.list(at)
}},
{Name: RouteTool,
Description: "One site the mesh's web front (route-proxy) is asked about, in full: every rule it is served by " +
"(path, module, node, target, refusal, redirect, authentication), its certificate (issuer, not-after, last " +
"renewal and its outcome, or why it failed) and the last upstream answer. A name with no route says so, and " +
"when it was last asked for. Answers is X reachable, why does this site not load. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
"name": map[string]any{"type": "string", "description": "the site's name, such as app.example.org"},
"node": node}},
Run: func(args map[string]any) (any, error) {
name, _ := args["name"].(string)
at, _ := args["node"].(string)
return f.one(name, at)
}},
{Name: UnroutedTool,
Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " +
"a TLS handshake that no route serves, newest first, with when it was last asked and how often. Only names under a " +
"domain the proxy serves are kept by name, a few new ones a minute; every other refusal (a scan) is counted per " +
"proxy, the last minute's count and the total since its start. A site that should be served and is missing " +
"its route shows here. Kept a week. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
Run: func(args map[string]any) (any, error) {
at, _ := args["node"].(string)
return f.refusals(at)
}},
}
}
// read is every value of a state, of node's proxy alone when node is named, decoded into a fresh T each.
func read[T any](state reader, node string) ([]T, error) {
keys, err := state.Keys()
if err != nil {
return nil, err
}
var out []T
for _, key := range keys {
at, _ := contract.Split(key)
if node != "" && at != node {
continue
}
entry, err := state.Get(key)
if err != nil {
return nil, err
}
if entry == nil {
continue // deleted between the list and the read
}
var v T
if err := json.Unmarshal(entry.Value, &v); err != nil {
return nil, fmt.Errorf("%s holds what is not a route-proxy record: %w", key, err)
}
out = append(out, v)
}
return out, nil
}
func (f front) list(node string) (map[string]any, error) {
routes, err := read[contract.Route](f.routes, node)
if err != nil {
return nil, err
}
lines := make([]RouteLine, 0, len(routes))
unreachable, expiring := 0, 0
for _, r := range routes {
line := f.line(r)
if line.Reachable == "no" {
unreachable++
}
if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == contract.Failed {
expiring++
}
lines = append(lines, line)
}
sort.Slice(lines, func(i, j int) bool {
if lines[i].Name != lines[j].Name {
return lines[i].Name < lines[j].Name
}
return lines[i].Proxy < lines[j].Proxy
})
return map[string]any{"routes": lines, "served": len(lines), "unreachable": unreachable,
"certificates_failing_or_within_14_days": expiring}, nil
}
func (f front) line(r contract.Route) RouteLine {
line := RouteLine{Name: r.Name, Proxy: r.Proxy, Reach: r.Reach, LastAnswer: r.LastAnswer, Reachable: "unknown",
Targets: []string{}}
for _, t := range r.Targets {
line.Targets = append(line.Targets, describe(t))
}
if a := r.LastAnswer; a != nil {
line.Reachable = "yes"
if a.Error != "" || a.Status >= 500 || a.Status == 0 {
line.Reachable = "no"
}
}
c := r.Certificate
words := []string{}
if c.Outcome != "" {
words = append(words, c.Outcome)
}
if c.Issuer != "" {
words = append(words, "by "+c.Issuer)
}
if c.NotAfter != nil {
days := int(c.NotAfter.Sub(f.now()).Hours() / 24)
line.ExpiresInDays = &days
words = append(words, "until "+c.NotAfter.UTC().Format("2006-01-02"))
}
if c.Error != "" {
words = append(words, "("+c.Error+")")
}
if len(words) == 0 {
words = append(words, "none seen yet")
}
line.Certificate = c.Authority + ": " + strings.Join(words, " ")
return line
}
// describe is one rule in a few words: path → module@node at target, or what it answers with.
func describe(t contract.Target) string {
path := t.Path
if path == "" {
path = "/"
}
var what string
switch {
case t.Deny:
what = "refused"
case t.Redirect != "":
what = "redirected to " + t.Redirect
default:
what = fmt.Sprintf("%s@%s at %s", t.Module, t.Node, t.Target)
}
if t.Sealed != "" {
what += " (authentication cannot be checked: " + t.Sealed + ")"
} else if t.Auth {
what += " (authenticated)"
}
return path + " → " + what
}
func (f front) one(name, node string) (map[string]any, error) {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return nil, fmt.Errorf("name is the site to read, such as app.example.org")
}
routes, err := read[contract.Route](f.routes, node)
if err != nil {
return nil, err
}
var found []contract.Route
for _, r := range routes {
if r.Name == name {
found = append(found, r)
}
}
if len(found) > 0 {
out := map[string]any{"name": name, "routed": true, "routes": found}
if len(found) == 1 {
out["reachable"] = f.line(found[0]).Reachable
}
return out, nil
}
asked, _, err := f.asked(node)
if err != nil {
return nil, err
}
for _, a := range asked {
if a.Name == name {
return map[string]any{"name": name, "routed": false, "asked": a,
"says": fmt.Sprintf("no proxy serves %s; the proxy on %s answered \"no route\" for it, last at %s",
name, a.Proxy, a.AskedAt.UTC().Format(time.RFC3339))}, nil
}
}
return map[string]any{"name": name, "routed": false,
"says": fmt.Sprintf("no proxy serves %s, and none was asked for it this week", name)}, nil
}
func (f front) refusals(node string) (map[string]any, error) {
asked, others, err := f.asked(node)
if err != nil {
return nil, err
}
sort.Slice(asked, func(i, j int) bool { return asked[i].AskedAt.After(asked[j].AskedAt) })
if asked == nil {
asked = []contract.Unrouted{}
}
return map[string]any{"unrouted": asked, "names": len(asked), "other_refusals": others}, nil
}
// asked is the names refused and kept by name, and each proxy's count of the refusals not kept by name.
func (f front) asked(node string) ([]contract.Unrouted, []contract.Other, error) {
keys, err := f.unrouted.Keys()
if err != nil {
return nil, nil, err
}
names, others := keySet{}, keySet{}
for _, key := range keys {
if _, name := contract.Split(key); name == contract.OtherName {
others[key] = true
} else {
names[key] = true
}
}
asked, err := read[contract.Unrouted](only{f.unrouted, names}, node)
if err != nil {
return nil, nil, err
}
counted, err := read[contract.Other](only{f.unrouted, others}, node)
if counted == nil {
counted = []contract.Other{}
}
return asked, counted, err
}
// keySet is a set of keys; only is a state narrowed to them.
type keySet map[string]bool
type only struct {
reader
keys keySet
}
func (o only) Keys() ([]string, error) {
out := make([]string, 0, len(o.keys))
for k := range o.keys {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
modules/route-proxy/cmd/route-provider/tools_test.go
-99
View File
@@ -1,99 +0,0 @@
package main
import (
"encoding/json"
"sort"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// held is a state as the bus holds it, read-only.
type held map[string]any
func (h held) Keys() ([]string, error) {
out := []string{}
for k := range h {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (h held) Get(key string) (*stdio.StateEntry, error) {
v, ok := h[key]
if !ok {
return nil, nil
}
raw, err := json.Marshal(v)
return &stdio.StateEntry{Key: key, Value: raw}, err
}
// The web front's tools answer from the bus, in one call each (issue 774): every site with its target, certificate
// expiry and last answer; one site in full, or that it is not served and when it was asked for; and the names the
// proxy answered "no route for".
func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) {
now := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
expires := now.Add(30 * 24 * time.Hour)
routes := held{
"anchor.grafana.example": contract.Route{Name: "grafana.example", Proxy: "anchor", Reach: contract.ReachPublic,
Targets: []contract.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}},
Certificate: contract.Certificate{Authority: contract.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: contract.Issued},
LastAnswer: &contract.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}},
"anchor.drive.example": contract.Route{Name: "drive.example", Proxy: "anchor", Reach: contract.ReachPublic,
Targets: []contract.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}},
Certificate: contract.Certificate{Authority: contract.ReachPublic},
LastAnswer: &contract.Answer{Status: 200, At: now}},
}
unrouted := held{
"anchor.old.example": contract.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3},
"anchor.wiki.example": contract.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1},
"anchor._other": contract.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900},
}
tools := map[string]stdio.Tool{}
for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() {
tools[tool.Name] = tool
}
for _, row := range []struct {
tool string
args map[string]any
want []string // what the answer says, in its JSON
}{
{RoutesTool, nil, []string{`"served":2`, `"unreachable":1`,
`"name":"grafana.example"`, `"reachable":"no"`, `"expires_in_days":30`, `/ → grafana@home-server at http://10.0.0.2:3000`,
`"name":"drive.example"`, `"reachable":"yes"`, `public: none seen yet`}},
{RoutesTool, map[string]any{"node": "home-server"}, []string{`"served":0`}},
{RouteTool, map[string]any{"name": "Grafana.Example"}, []string{`"routed":true`, `"reachable":"no"`,
`connection refused`, `"issuer":"Let's Encrypt R11"`}},
{RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`,
`answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}},
{RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}},
{UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`,
`"other_refusals":[{"proxy":"anchor"`, `"last_minute":40`, `"since_start":900`}},
} {
t.Run(row.tool, func(t *testing.T) {
tool, ok := tools[row.tool]
if !ok {
t.Fatalf("%s is not served", row.tool)
}
if !strings.Contains(tool.Description, "route-proxy") {
t.Errorf("%s's description does not name the web front, so a search does not find it", row.tool)
}
answer, err := tool.Run(row.args)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(answer)
for _, w := range row.want {
if !strings.Contains(string(raw), w) {
t.Errorf("%s(%v) does not say %s:\n%s", row.tool, row.args, w, raw)
}
}
})
}
}
modules/route-proxy/cmd/route-proxy/authority_test.go
-75
View File
@@ -1,75 +0,0 @@
package main
import (
"path/filepath"
"strings"
"testing"
)
// The lab's first root, and the one a re-initialised CA generates in its place.
const (
firstRoot = "-----BEGIN CERTIFICATE-----\nMIIBeFIRST\n-----END CERTIFICATE-----\n"
secondRoot = "-----BEGIN CERTIFICATE-----\nMIIBeSECOND\n-----END CERTIFICATE-----\n"
)
// A re-initialised CA does not need somebody to delete the cache by hand.
//
// autocert keeps its account key at one fixed name and reuses it for ever. When an internal CA is
// re-initialised it has never heard of that account, rejects every use of it, and autocert has no
// path back: nothing is re-registered, no order reaches the CA, and issuance stops with nothing
// saying why. Naming the cache after the authority means the account is only ever found where it is
// still valid — the new root lands in a directory with no account in it, and autocert registers.
func TestTheAccountCacheIsNamedAfterItsAuthority(t *testing.T) {
t.Run("a new CA root means a new account cache", func(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot))
if before == after {
t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before)
}
})
// And the SAME authority keeps the account it registered, restart after restart.
//
// This is the whole reason ACME_CACHE is required in the first place: an account and its
// certificates that did not persist would be re-ordered on every restart, which works silently until
// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the
// mesh writes that file, and a trailing newline coming or going must not throw away an account.
t.Run("the same authority keeps its account", func(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n"))
if first != again {
t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s",
first, again)
}
})
// Staging and production are different authorities, and were sharing one account.
//
// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against
// staging reused the staging account, because the cache had no idea they were different.
t.Run("staging and production do not share an account", func(t *testing.T) {
staging := forThisAuthority("/acme", stagingDirectory, nil)
production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil)
if staging == production {
t.Errorf("two issuers share one account: %s", staging)
}
})
// It stays inside the directory the mesh gave it, and is a plain name.
//
// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or
// that carried a separator out of the PEM — would put an account somewhere nothing persists.
t.Run("the account cache stays where the mesh put it", func(t *testing.T) {
const cache = "/var/lib/route-proxy/acme"
got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot))
if !strings.HasPrefix(got, cache+"/") {
t.Fatalf("the account cache is not under %s: %s", cache, got)
}
name := strings.TrimPrefix(got, cache+"/")
if name != filepath.Base(got) || strings.ContainsAny(name, "/.") {
t.Errorf("the account cache is not a plain name: %q", name)
}
})
}
modules/route-proxy/cmd/route-proxy/bus.go
-194
View File
@@ -1,194 +0,0 @@
package main
import (
"fmt"
"log"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// keptStates is every state route-proxy keeps, as both its halves write them: the routes and the refusals this
// proxy writes, readable by others, and the bound entries the provider helper writes.
func keptStates() []announce.State {
return []announce.State{
{Name: contract.BoundState},
{Name: contract.RoutesState, Readable: true},
{Name: contract.UnroutedState, Readable: true},
}
}
// What the mesh says this proxy routes, and who is inside the mesh, read on the bus.
//
// **The routes are the seat mesh-provisioner's plans, alone** (issue 774, research 072 row 34): every plan's entry
// naming this proxy as a route's provider, followed, so a route added or withdrawn reaches a running proxy without a
// restart, and said on the bus as it changes (contract.EventServed, EventWithdrawn, EventRefused). **Who is inside**
// is every node's address on the private network as the seat mesh-placement decides it (mesh-sdk
// delivery.InsideAddresses), which is who may be served an internal name; until it is read, an internal name is
// served to this machine alone.
// follow connects with the credential in path, follows the plans and the placement, and retries the connection for
// as long as it takes: until the bus answers, the proxy routes nothing, and says so.
func follow(path string, held *table) {
for {
err := followOnce(path, held)
if err == nil {
return
}
log.Printf("cannot follow the plans that say what this proxy routes, so it routes what it last read: %v", err)
time.Sleep(30 * time.Second)
}
}
func followOnce(path string, held *table) error {
account, err := process.ReadAccount(path)
if err != nil {
return err
}
conn, err := process.Connect(account, nats.Name(account.Node+"."+account.Module))
if err != nil {
return fmt.Errorf("connecting to the bus at %s: %w", membership.BareAddress(account.URL), err)
}
// What this proxy keeps, said on the bus (novox/hq ADR 0495; issue 774): the names it serves and the names it
// refused, readable by any module granted a read of them; the provider helper's entries, read by the machines'
// clients through their own grant. Said before the first write, so no write of an unannounced state is refused.
if err := conn.Keeps(keptStates()...); err != nil {
log.Printf("cannot say on the bus what this proxy keeps, so it keeps nothing there: %v", err)
} else {
held.ledger.attach(account.Node, conn.State(contract.RoutesState), conn.State(contract.UnroutedState))
}
say := func(eventType string, e contract.Event) {
if err := conn.Emit(eventType, e); err != nil {
log.Printf("cannot say %s of %q on the bus: %v", eventType, e.Name, err)
}
}
// Who is inside first, so the first table is served to the machines it is meant for.
if err := followInside(conn.Read(delivery.PlacementRead), held); err != nil {
log.Printf("cannot follow %s, so an internal name is served to this machine alone: %v", delivery.PlacementRead, err)
}
if err := followPlans(conn.Read(provisions.PlansRead), newPlanned(account.Node, say), held, say); err != nil {
conn.Close()
return err
}
return nil
}
// planWatch is a state this proxy watches: every current key, then every change.
type planWatch interface {
Watch(key string, handle func(stdio.StateChange) error) error
}
// followPlans serves what the plans give this proxy: the whole table once every current plan is read, then again on
// every change that alters it. A table built from part of the plans would withdraw every route not yet read.
func followPlans(plans planWatch, given *planned, held *table, say func(string, contract.Event)) error {
var read atomic.Bool // every current plan is read: from here, each change is served as it comes
err := plans.Watch("", func(c stdio.StateChange) error {
var value []byte
if c.Op != "delete" && c.Op != "purge" {
value = c.Value
}
if changed := given.take(c.Key, value); changed && read.Load() {
serve(given, held, say)
}
return nil
})
if err != nil {
return fmt.Errorf("watching %s: %w", provisions.PlansRead, err)
}
read.Store(true)
serve(given, held, say)
return nil
}
// serve replaces what the proxy serves with what the plans give it, and says each name it now serves or serves no
// more.
func serve(given *planned, held *table, say func(string, contract.Event)) {
before := held.served()
routes, public := routesOf(given.contributions())
held.set(routes, public)
after := held.served()
for _, e := range servedChanges(given.node, before, after) {
say(e.kind, e.Event)
}
log.Printf("serving %d route(s) from the seat's plans: %s", len(routes), strings.Join(held.names(), ", "))
}
// change is one event serve says.
type change struct {
kind string
contract.Event
}
// servedChanges is what changed between two tables' names, each name once, in order: a name now served, or now
// served at another reach or by another module, is served; a name no longer served is withdrawn.
func servedChanges(proxy string, before, after map[string]contract.Event) []change {
var out []change
for _, name := range sortedNames(after) {
if was, ok := before[name]; !ok || was != after[name] {
e := after[name]
e.Proxy = proxy
out = append(out, change{contract.EventServed, e})
}
}
for _, name := range sortedNames(before) {
if _, ok := after[name]; !ok {
out = append(out, change{contract.EventWithdrawn, contract.Event{Proxy: proxy, Name: name}})
}
}
return out
}
func sortedNames(m map[string]contract.Event) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// followInside follows the seat mesh-placement's state and serves an internal name to every node's address on the
// private network it decides, each time it changes (mesh-sdk delivery.InsideAddresses; research 072 rows 33 and 34:
// the membership that carried them went with the assignments stream). A key that does not read keeps who is inside
// as it was.
func followInside(placement planWatch, held *table) error {
keys := delivery.NewPlacementKeys()
var read atomic.Bool
apply := func() {
inside, err := sourcesOf(delivery.InsideAddresses(keys.All()))
if err != nil {
log.Printf("the placement says an address that is not one, so who is inside stays as it was: %v", err)
return
}
held.setInside(inside)
log.Printf("internal names are served to %d machine(s)", len(inside))
}
err := placement.Watch("", func(c stdio.StateChange) error {
if _, err := keys.Put(c.Key, c.Value, c.Revision, c.Op == "delete" || c.Op == "purge"); err != nil {
log.Printf("the placement's key %s does not read, so who is inside stays as it was: %v", c.Key, err)
return nil
}
if read.Load() {
apply()
}
return nil
})
if err != nil {
return err
}
read.Store(true)
apply()
return nil
}
modules/route-proxy/cmd/route-proxy/challenge_test.go
-109
View File
@@ -1,109 +0,0 @@
package main
// The challenge path falls through for real. autocert's own HTTPHandler answers 404 itself for a
// token it does not hold and never consults its fallback on the challenge path — the
// predecessor's fault, the edge owning /.well-known/acme-challenge outright, rediscovered live
// when Mailu's renewal died behind this proxy on cutover day (2026-09-26). These tests pin the
// three behaviours tokenOrRoute exists for.
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"golang.org/x/crypto/acme/autocert"
)
func routedTo(t *testing.T, marker string) http.Handler {
t.Helper()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
if _, err := w.Write([]byte(marker)); err != nil {
t.Fatal(err)
}
})
}
func TestATokenNoAuthorityHoldsIsRoutedNot404d(t *testing.T) {
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
h := tokenOrRoute(routedTo(t, "the workload answered"), m)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/somebody-elses-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
t.Fatalf("a token no authority holds must reach plain routing; got %d %q", rec.Code, rec.Body.String())
}
}
func TestATokenAManagerHoldsIsAnsweredByIt(t *testing.T) {
// autocert reads a token it does not have in memory from its cache, under "<token>+http-01" —
// which is also how a token would survive the manager restarting mid-issuance.
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "held-token+http-01"), []byte("the-key-authorization"), 0o600); err != nil {
t.Fatal(err)
}
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
h := tokenOrRoute(routedTo(t, "must not be reached"), m)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/held-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "the-key-authorization" {
t.Fatalf("the manager holding a token answers it; got %d %q", rec.Code, rec.Body.String())
}
}
func TestASecondAuthorityIsProbedBeforeRouting(t *testing.T) {
first := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "internal-token+http-01"), []byte("internal-key"), 0o600); err != nil {
t.Fatal(err)
}
second := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
h := tokenOrRoute(routedTo(t, "must not be reached"), first, second)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://git.internal/.well-known/acme-challenge/internal-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "internal-key" {
t.Fatalf("the second authority's token is found by probing past the first; got %d %q", rec.Code, rec.Body.String())
}
}
func TestAnAuthorityWhosePolicyRefusesTheNameIsProbedPast(t *testing.T) {
// autocert checks the host policy before the token and answers 403 — the internal authority
// does this for every public name. A policy refusal is as much "not mine" as a missing token:
// the request must still reach plain routing, where the workload's own ACME client answers.
refusing := &autocert.Manager{
Prompt: autocert.AcceptTOS,
Cache: autocert.DirCache(t.TempDir()),
HostPolicy: func(ctx context.Context, host string) error {
return fmt.Errorf("no internal-only route for %q in this mesh", host)
},
}
h := tokenOrRoute(routedTo(t, "the workload answered"), refusing)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/mailus-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
t.Fatalf("a policy refusal must fall through to routing; got %d %q", rec.Code, rec.Body.String())
}
}
func TestAnOrdinaryPathNeverTouchesTheChallengeMachinery(t *testing.T) {
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
h := tokenOrRoute(routedTo(t, "routed"), m)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://site.example/index.html", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "routed" {
t.Fatalf("an ordinary path goes straight to routing; got %d %q", rec.Code, rec.Body.String())
}
}
modules/route-proxy/cmd/route-proxy/forwarded_test.go
-29
View File
@@ -1,29 +0,0 @@
package main
import (
"crypto/tls"
"net/http"
"net/http/httptest"
"net/url"
"testing"
)
// A backend behind the proxy learns the client used TLS and which name it asked for, so the addresses
// it writes into its own pages are the ones a client can use (2026-10-03: a forge's Go import tag
// named an http clone URL, and Go refused the module path).
func TestABackendIsToldTheRequestWasHTTPSAndForWhichName(t *testing.T) {
var proto, host, fwdHost, fwdFor string
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
proto, host, fwdHost, fwdFor = r.Header.Get("X-Forwarded-Proto"), r.Host, r.Header.Get("X-Forwarded-Host"), r.Header.Get("X-Forwarded-For")
}))
defer backend.Close()
where, _ := url.Parse(backend.URL)
req := httptest.NewRequest(http.MethodGet, "https://git.example.org/novox/mesh-sdk/go?go-get=1", nil)
req.TLS = &tls.ConnectionState{}
req.Host = "git.example.org"
req.RemoteAddr = "192.0.2.7:51000"
towards(where).ServeHTTP(httptest.NewRecorder(), req)
if proto != "https" || fwdHost != "git.example.org" || host != "git.example.org" || fwdFor != "192.0.2.7" {
t.Errorf("the backend was told proto=%q host=%q forwarded-host=%q for=%q", proto, host, fwdHost, fwdFor)
}
}
modules/route-proxy/cmd/route-proxy/ledger.go
-552
View File
@@ -1,552 +0,0 @@
package main
import (
"context"
"crypto/x509"
"encoding/json"
"encoding/pem"
"log"
"net/http"
"reflect"
"sort"
"strings"
"sync"
"time"
"golang.org/x/crypto/acme/autocert"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// What this proxy says on the bus about the names it serves and the names it refuses (issue 774).
//
// **On change, never on a timer.** A route arriving or leaving, a certificate issued or failing, an upstream answer
// that differs from the last one written, a name asked for that is not served: each marks its key, and one writer
// puts the marked keys. An answer that is the same as the last one written is written again only once it is older
// than refreshAfter, so a site answering 200 all day costs one write a minute, not one per request.
//
// Until the bus is reached nothing is written; what happened meanwhile is held and written when it is.
// refreshAfter is how old a written answer, refusal or certificate failure may be before the same one is written
// again with its new time.
const refreshAfter = time.Minute
// mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes.
const mostAsked = 4096
// newPerMinute is how many names not seen before are kept by name in a minute; past it they are only counted.
const newPerMinute = 5
// kvStore is one of the module's states, as process.State is one.
type kvStore interface {
Put(key string, value any) (uint64, error)
Delete(key string) error
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// ledger is what this proxy keeps of what it serves and refuses, and writes to the bus.
type ledger struct {
mu sync.Mutex
now func() time.Time
node string
routes, unrouted kvStore
served map[string]*contract.Route
previous map[string]contract.Route // what the bus held for this machine when it was reached
asked map[string]*contract.Unrouted
answerWritten map[string]time.Time
askedWritten map[string]time.Time
dirtyRoutes map[string]bool // a name to put when served, to delete when not
dirtyAsked map[string]bool
// domains are the registrable domains of the names served: a refused name under one is kept by name.
domains map[string]bool
newMinute time.Time
newInMinute int
other contract.Other
otherMinute time.Time
otherInMinute int
otherDirty bool
routesFailing, askFailed bool // a failed write said once, until one succeeds
kick chan struct{}
}
func newLedger() *ledger {
return &ledger{
now: time.Now,
served: map[string]*contract.Route{},
previous: map[string]contract.Route{},
asked: map[string]*contract.Unrouted{},
answerWritten: map[string]time.Time{},
askedWritten: map[string]time.Time{},
dirtyRoutes: map[string]bool{},
dirtyAsked: map[string]bool{},
domains: map[string]bool{},
kick: make(chan struct{}, 1),
}
}
// loop writes whatever was marked, each time something was.
func (l *ledger) loop() {
for range l.kick {
l.flush()
}
}
func (l *ledger) wake() {
select {
case l.kick <- struct{}{}:
default:
}
}
// attach is the bus reached, as node: what the bus held for this machine is read back, so a restart keeps a name's
// certificate and last answer, and everything held is written.
func (l *ledger) attach(node string, routes, unrouted kvStore) {
if l == nil {
return
}
before := map[string]contract.Route{}
if keys, err := routes.Keys(); err != nil {
log.Printf("cannot read back what this proxy said it serves: %v", err)
} else {
for _, key := range keys {
at, name := contract.Split(key)
if at != node {
continue
}
entry, err := routes.Get(key)
if err != nil || entry == nil {
continue
}
var r contract.Route
if json.Unmarshal(entry.Value, &r) == nil {
before[name] = r
}
}
}
l.mu.Lock()
l.node, l.routes, l.unrouted = node, routes, unrouted
for name, r := range before {
l.previous[name] = r
l.dirtyRoutes[name] = true // put again when served, deleted when no longer
if now := l.served[name]; now != nil {
keepWhatWasKnown(now, r)
}
}
for name := range l.served {
l.dirtyRoutes[name] = true
}
for name := range l.asked {
l.dirtyAsked[name] = true
}
l.mu.Unlock()
l.wake()
}
// keepWhatWasKnown carries a certificate and a last answer read back from the bus onto a route that knows none yet.
func keepWhatWasKnown(r *contract.Route, before contract.Route) {
if r.Certificate.Outcome == "" && r.Certificate.NotAfter == nil && before.Certificate.Authority == r.Certificate.Authority {
r.Certificate = before.Certificate
}
if r.LastAnswer == nil {
r.LastAnswer = before.LastAnswer
}
}
// serving is the table this proxy now serves: a name new or changed is written, a name gone is deleted, and a name
// that was asked for unserved and now is served leaves the unrouted state.
func (l *ledger) serving(routes map[string][]rule, public map[string]bool) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
for name, rules := range routes {
reach := contract.ReachInternal
if public[name] {
reach = contract.ReachPublic
}
targets := targetsOf(rules)
r := l.served[name]
if r == nil {
r = &contract.Route{Name: name}
if before, ok := l.previous[name]; ok {
r.Certificate.Authority = reach
keepWhatWasKnown(r, before)
}
l.served[name] = r
l.dirtyRoutes[name] = true
}
if r.Reach != reach || !reflect.DeepEqual(r.Targets, targets) {
r.Reach, r.Targets = reach, targets
l.dirtyRoutes[name] = true
}
if r.Certificate.Authority != reach {
r.Certificate = contract.Certificate{Authority: reach}
l.dirtyRoutes[name] = true
}
if _, was := l.asked[name]; was {
delete(l.asked, name)
l.dirtyAsked[name] = true
}
}
l.domains = map[string]bool{}
for name := range routes {
if d := contract.Registrable(name); d != "" {
l.domains[d] = true
}
}
for name := range l.served {
if _, still := routes[name]; !still {
delete(l.served, name)
delete(l.answerWritten, name)
l.dirtyRoutes[name] = true
}
}
l.wake()
}
func targetsOf(rules []rule) []contract.Target {
out := make([]contract.Target, 0, len(rules))
for _, r := range rules {
out = append(out, contract.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target,
Deny: r.policy.deny, Redirect: r.policy.redirectTo, Auth: len(r.policy.users) > 0 || r.policy.sealed != "",
Sealed: r.policy.sealed})
}
return out
}
// answered is an upstream answer to a request for name: its status, or the error that stood in for one.
func (l *ledger) answered(name, path string, status int, failed error) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
why := ""
if failed != nil {
why = failed.Error()
}
before := r.LastAnswer
r.LastAnswer = &contract.Answer{Status: status, At: now, Error: why, Path: path}
if before != nil && before.Status == status && before.Error == why && now.Sub(l.answerWritten[name]) < refreshAfter {
return
}
l.answerWritten[name] = now
l.dirtyRoutes[name] = true
l.wake()
}
// refused is a name this proxy answered "no route for": in a request, or in a handshake's server name.
//
// **Only a name under a domain this proxy serves is kept by name**, and at most newPerMinute new ones a minute: a scan
// sends any name it likes, and each kept name is a key on the bus. Every other refusal is counted, and the count is
// written once a minute under the proxy's OtherName key. Nothing here scans: forgetting old names is the writer's.
// Who asked is not kept: the journal line beside it is what the machine's intrusion prevention reads.
func (l *ledger) refused(name string, internalOnly, handshake bool) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
if _, isServed := l.served[name]; isServed && !internalOnly {
return
}
now := l.now()
a := l.asked[name]
if a == nil {
if !contract.Keyable(name) || !l.domains[contract.Registrable(name)] || !l.mayAddName(now) {
l.countOther(now)
return
}
a = &contract.Unrouted{Name: name}
l.asked[name] = a
}
a.AskedAt, a.InternalOnly, a.Handshake = now, internalOnly, handshake
a.Times++
if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter {
return
}
l.askedWritten[name] = now
l.dirtyAsked[name] = true
l.wake()
}
// mayAddName says whether another new name may be kept this minute, and counts it.
func (l *ledger) mayAddName(now time.Time) bool {
if minute := now.Truncate(time.Minute); !minute.Equal(l.newMinute) {
l.newMinute, l.newInMinute = minute, 0
}
if l.newInMinute >= newPerMinute {
return false
}
l.newInMinute++
return true
}
// countOther counts a refusal not kept by name. A minute's count is written when the next minute's first one comes.
func (l *ledger) countOther(now time.Time) {
minute := now.Truncate(time.Minute)
if !minute.Equal(l.otherMinute) {
if l.otherInMinute > 0 {
l.other.Minute, l.other.LastMinute = l.otherMinute, l.otherInMinute
l.otherDirty = true
l.wake()
}
l.otherMinute, l.otherInMinute = minute, 0
}
l.otherInMinute++
l.other.SinceStart++
}
// forgetOldestAsked drops the names asked longest ago from memory until mostAsked are left; the bus keeps them until
// their time to live ends. Called by the writer, never on a request.
func (l *ledger) forgetOldestAsked() {
if len(l.asked) <= mostAsked {
return
}
names := make([]string, 0, len(l.asked))
for name := range l.asked {
names = append(names, name)
}
sort.Slice(names, func(i, j int) bool { return l.asked[names[i]].AskedAt.Before(l.asked[names[j]].AskedAt) })
for _, name := range names[:len(names)-mostAsked] {
delete(l.asked, name)
delete(l.askedWritten, name)
delete(l.dirtyAsked, name)
}
}
// certified is a certificate an authority issued for name, as autocert stored it.
func (l *ledger) certified(name string, leaf *x509.Certificate) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
notAfter := leaf.NotAfter.UTC()
r.Certificate.Issuer, r.Certificate.NotAfter = issuerOf(leaf), &notAfter
r.Certificate.RenewedAt, r.Certificate.At = &now, &now
r.Certificate.Outcome, r.Certificate.Error = contract.Issued, ""
l.dirtyRoutes[name] = true
l.wake()
}
// handedOut is the certificate a handshake for name was answered with: written when it is not the one recorded.
func (l *ledger) handedOut(name string, leaf *x509.Certificate) {
if l == nil || leaf == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
notAfter := leaf.NotAfter.UTC()
if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != contract.Failed {
return
}
now := l.now()
r.Certificate.Issuer, r.Certificate.NotAfter, r.Certificate.At = issuerOf(leaf), &notAfter, &now
if r.Certificate.Outcome != contract.Issued || r.Certificate.Error != "" {
r.Certificate.Outcome, r.Certificate.Error = contract.Cached, ""
}
l.dirtyRoutes[name] = true
l.wake()
}
// certificateFailed is a handshake for name that found no certificate to answer with, and why.
func (l *ledger) certificateFailed(name string, failed error) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
same := r.Certificate.Outcome == contract.Failed && r.Certificate.Error == failed.Error()
if same && r.Certificate.At != nil && now.Sub(*r.Certificate.At) < refreshAfter {
return
}
r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = contract.Failed, failed.Error(), &now
l.dirtyRoutes[name] = true
l.wake()
}
func issuerOf(leaf *x509.Certificate) string {
if cn := leaf.Issuer.CommonName; cn != "" {
if len(leaf.Issuer.Organization) > 0 {
return leaf.Issuer.Organization[0] + " " + cn
}
return cn
}
return leaf.Issuer.String()
}
// flush writes every marked key. A route's write that fails stays marked and is tried again at the next change; a
// refused name's write that fails is dropped, never tried again: the bucket refusing it is full, or the bus is away,
// and either way it is not worth a storm of retries. Each kind of failure is logged once, until a write succeeds.
func (l *ledger) flush() {
type put struct {
store kvStore
key string
value any // nil deletes
route bool
}
l.mu.Lock()
if l.routes == nil {
l.mu.Unlock()
return
}
l.forgetOldestAsked()
var puts []put
for name := range l.dirtyRoutes {
p := put{store: l.routes, key: contract.Key(l.node, name), route: true}
if r := l.served[name]; r != nil {
c := *r
c.Proxy = l.node
p.value = c
}
puts = append(puts, p)
}
for name := range l.dirtyAsked {
p := put{store: l.unrouted, key: contract.Key(l.node, name)}
if a := l.asked[name]; a != nil {
c := *a
c.Proxy = l.node
p.value = c
}
puts = append(puts, p)
}
if l.otherDirty {
o := l.other
o.Proxy = l.node
puts = append(puts, put{store: l.unrouted, key: contract.Key(l.node, contract.OtherName), value: o})
}
l.dirtyRoutes, l.dirtyAsked, l.otherDirty = map[string]bool{}, map[string]bool{}, false
l.mu.Unlock()
for _, p := range puts {
var err error
if p.value == nil {
err = p.store.Delete(p.key)
} else {
_, err = p.store.Put(p.key, p.value)
}
l.mu.Lock()
switch {
case err == nil && p.route:
l.routesFailing = false
case err == nil:
l.askFailed = false
case p.route:
if !l.routesFailing {
log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err)
}
l.routesFailing = true
_, name := contract.Split(p.key)
l.dirtyRoutes[name] = true
default:
if !l.askFailed {
log.Printf("cannot write %s to the bus, dropped with every refused name until a write succeeds: %v",
p.key, err)
}
l.askFailed = true
}
l.mu.Unlock()
}
}
// recordingCache is an authority's certificate cache that tells the ledger of every certificate stored in it: the
// one place autocert says it was issued one.
type recordingCache struct {
autocert.Cache
held *table
}
func (c recordingCache) Put(ctx context.Context, key string, data []byte) error {
if err := c.Cache.Put(ctx, key, data); err != nil {
return err
}
name := strings.TrimSuffix(key, "+rsa")
if strings.Contains(name, "+") || !c.held.routed(name) {
return nil // the account key, a challenge token
}
if leaf := leafOf(data); leaf != nil {
c.held.ledger.certified(bareHost(name), leaf)
}
return nil
}
// leafOf is the first certificate in what autocert stores: the private key, then the chain, in PEM.
func leafOf(data []byte) *x509.Certificate {
for {
var block *pem.Block
block, data = pem.Decode(data)
if block == nil {
return nil
}
if block.Type == "CERTIFICATE" {
leaf, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil
}
return leaf
}
}
}
// answerRecorder carries a proxied request's answer through and remembers its status and the proxy's error.
type answerRecorder struct {
http.ResponseWriter
status int
err error
}
func (a *answerRecorder) WriteHeader(status int) {
if a.status == 0 {
a.status = status
}
a.ResponseWriter.WriteHeader(status)
}
func (a *answerRecorder) Write(b []byte) (int, error) {
if a.status == 0 {
a.status = http.StatusOK
}
return a.ResponseWriter.Write(b)
}
// Unwrap lets http.ResponseController reach the connection beneath: a flush, an upgrade's hijack.
func (a *answerRecorder) Unwrap() http.ResponseWriter { return a.ResponseWriter }
// proxyFailed is the reverse proxy's answer when upstream gave none: the error kept for the ledger, and a 502.
func proxyFailed(w http.ResponseWriter, r *http.Request, err error) {
if a, ok := w.(*answerRecorder); ok {
a.err = err
}
log.Printf("http: proxy error for %s%s: %v", r.Host, r.URL.Path, err)
w.WriteHeader(http.StatusBadGateway)
}
modules/route-proxy/cmd/route-proxy/ledger_test.go
-291
View File
@@ -1,291 +0,0 @@
package main
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"net/http"
"net/http/httptest"
"sort"
"strings"
"testing"
"time"
"golang.org/x/crypto/acme/autocert"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// fakeState is a state on the bus, in memory, counting what was written to it.
type fakeState struct {
values map[string]json.RawMessage
puts int
tries int // every Put asked, refused or not
fail error // what every Put is refused with, while set
}
func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} }
func (f *fakeState) Put(key string, value any) (uint64, error) {
f.tries++
if f.fail != nil {
return 0, f.fail
}
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
f.values[key] = raw
f.puts++
return uint64(f.puts), nil
}
func (f *fakeState) Delete(key string) error { delete(f.values, key); return nil }
func (f *fakeState) Keys() ([]string, error) {
out := []string{}
for k := range f.values {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (f *fakeState) Get(key string) (*stdio.StateEntry, error) {
v, ok := f.values[key]
if !ok {
return nil, nil
}
return &stdio.StateEntry{Key: key, Value: v}, nil
}
func (f *fakeState) route(t *testing.T, key string) *contract.Route {
t.Helper()
raw, ok := f.values[key]
if !ok {
return nil
}
var r contract.Route
if err := json.Unmarshal(raw, &r); err != nil {
t.Fatal(err)
}
return &r
}
// issuedPEM is what autocert stores for a name: the key, then the certificate, signed by an issuer named here.
func issuedPEM(t *testing.T, name string, notAfter time.Time) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{SerialNumber: big.NewInt(1), DNSNames: []string{name},
Subject: pkix.Name{CommonName: name}, Issuer: pkix.Name{CommonName: "R11", Organization: []string{"Let's Encrypt"}},
NotBefore: notAfter.Add(-90 * 24 * time.Hour), NotAfter: notAfter}
issuer := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: template.Issuer, NotBefore: template.NotBefore,
NotAfter: notAfter, IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
keyDER, _ := x509.MarshalECPrivateKey(key)
out := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
return append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
}
// What the proxy tells the bus (issue 774): each served name with where it goes, its certificate and the last
// upstream answer; each name refused; and only on change. Without the ledger nothing reaches the bus, and the
// operator's question "is grafana reachable" is answered with curl.
func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) {
notAfter := time.Date(2027, 1, 28, 12, 0, 0, 0, time.UTC)
gone := func() string { // an address nothing listens on: the upstream that does not answer
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
return l.Addr().String()
}()
_, port, _ := net.SplitHostPort(gone)
contributions := []contribution{
{From: "grafana", Node: "anchor", At: "127.0.0.1", Route: contract.Request{Name: "grafana.example", Port: int(mustFloat(t, port))}},
{From: "drive", Node: "home-server", At: "10.0.0.2", Route: contract.Request{InternalName: "drive.internal", Port: 8080}},
}
for _, row := range []struct {
name string
act func(t *testing.T, held *table)
check func(t *testing.T, routes, unrouted *fakeState)
}{
{"a served name is written with the module and node it forwards to",
func(t *testing.T, held *table) {},
func(t *testing.T, routes, _ *fakeState) {
r := routes.route(t, "anchor.grafana.example")
if r == nil || r.Proxy != "anchor" || r.Reach != contract.ReachPublic || len(r.Targets) != 1 ||
r.Targets[0].Module != "grafana" || r.Targets[0].Node != "anchor" || r.Targets[0].Target != "http://127.0.0.1:"+port {
t.Fatalf("grafana's route is not on the bus as served: %+v", r)
}
if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != contract.ReachInternal ||
d.Certificate.Authority != contract.ReachInternal {
t.Fatalf("an internal-only name is not on the bus as internal: %+v", d)
}
}},
{"a withdrawn route is deleted",
func(t *testing.T, held *table) {
routes, public := routesOf(contributions[1:])
held.set(routes, public)
},
func(t *testing.T, routes, _ *fakeState) {
if routes.route(t, "anchor.grafana.example") != nil {
t.Fatal("a route no longer served is still on the bus")
}
}},
{"an upstream that does not answer is written as a 502 with why, once a minute",
func(t *testing.T, held *table) {
for range 3 {
req := httptest.NewRequest(http.MethodGet, "http://grafana.example/login", nil)
handler(held).ServeHTTP(httptest.NewRecorder(), req)
}
},
func(t *testing.T, routes, _ *fakeState) {
r := routes.route(t, "anchor.grafana.example")
if r == nil || r.LastAnswer == nil || r.LastAnswer.Status != http.StatusBadGateway ||
r.LastAnswer.Error == "" || r.LastAnswer.Path != "/login" {
t.Fatalf("the failed answer is not on the bus: %+v", r)
}
if routes.puts != 3 { // the two routes, then the first answer; the same answer again is not written
t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts)
}
}},
{"a name under a served domain is written as asked, without who asked, and leaves when it is served",
func(t *testing.T, held *table) {
req := httptest.NewRequest(http.MethodGet, "http://wiki.grafana.example/", nil)
req.RemoteAddr = "192.0.2.7:51000"
handler(held).ServeHTTP(httptest.NewRecorder(), req)
held.ledger.flush()
var a contract.Unrouted
raw := held.ledger.unrouted.(*fakeState).values["anchor.wiki.grafana.example"]
if err := json.Unmarshal(raw, &a); err != nil || a.Times != 1 || a.AskedAt.IsZero() {
t.Fatalf("the refused name is not on the bus as asked: %s", raw)
}
if strings.Contains(string(raw), "192.0.2.7") || strings.Contains(string(raw), `"from"`) {
t.Fatalf("who asked is on the bus: %s", raw)
}
routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor",
Route: contract.Request{Name: "wiki.grafana.example", Port: 3000}}))
held.set(routes, public)
},
func(t *testing.T, _, unrouted *fakeState) {
if len(unrouted.values) != 0 {
t.Fatalf("a name now served is still written as unrouted: %v", unrouted.values)
}
}},
{"a handshake for a name not routed is written as asked",
func(t *testing.T, held *table) {
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return nil, errors.New("no public route for \"forgotten.grafana.example\" in this mesh, so no certificate is asked for")
}, nil)
_, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.grafana.example"})
},
func(t *testing.T, _, unrouted *fakeState) {
var a contract.Unrouted
if err := json.Unmarshal(unrouted.values["anchor.forgotten.grafana.example"], &a); err != nil || !a.Handshake {
t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values)
}
}},
{"a flood of names writes at most the limit by name, counts the rest once a minute, and never retries a refused write",
func(t *testing.T, held *table) {
clock := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
held.ledger.now = func() time.Time { return clock }
full := held.ledger.unrouted.(*fakeState)
for i := range 40 {
held.ledger.refused(fmt.Sprintf("x%d.grafana.example", i), false, false) // under a served domain
held.ledger.refused(fmt.Sprintf("x%d.scan.test", i), false, true) // under none
}
held.ledger.flush()
if named := len(full.values); named != newPerMinute {
t.Fatalf("a flood wrote %d names, want the limit %d: %v", named, newPerMinute, full.values)
}
full.fail = errors.New("nats: maximum bytes exceeded")
clock = clock.Add(time.Minute)
held.ledger.refused("y.grafana.example", false, false) // a new minute: one more name,
held.ledger.refused("v.scan.test", false, false) // and the last minute's count
held.ledger.flush()
tries := full.tries
held.ledger.refused("y.grafana.example", false, false)
held.ledger.refused("z.scan.test", false, false)
held.ledger.flush()
if full.tries != tries {
t.Fatalf("writes the bucket refused were tried again: %d tries, then %d", tries, full.tries)
}
full.fail = nil
clock = clock.Add(time.Minute)
held.ledger.refused("w.scan.test", false, false)
},
func(t *testing.T, _, unrouted *fakeState) {
var o contract.Other
if err := json.Unmarshal(unrouted.values["anchor._other"], &o); err != nil || o.LastMinute != 2 || o.SinceStart != 78 {
t.Fatalf("the refusals not kept by name are not counted on the bus: %s", unrouted.values["anchor._other"])
}
}},
{"a certificate an authority issued is written with its issuer, expiry and renewal",
func(t *testing.T, held *table) {
cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held}
if err := cache.Put(context.Background(), "grafana.example", issuedPEM(t, "grafana.example", notAfter)); err != nil {
t.Fatal(err)
}
_ = cache.Put(context.Background(), "acme_account+key", []byte("not a certificate"))
},
func(t *testing.T, routes, _ *fakeState) {
c := routes.route(t, "anchor.grafana.example").Certificate
if c.Outcome != contract.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil ||
!c.NotAfter.Equal(notAfter) || c.RenewedAt == nil {
t.Fatalf("the issued certificate is not on the bus: %+v", c)
}
}},
{"a certificate that cannot be had is written with why",
func(t *testing.T, held *table) {
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return nil, errors.New("acme: urn:ietf:params:acme:error:rateLimited")
}, nil)
_, _ = pick(&tls.ClientHelloInfo{ServerName: "grafana.example"})
},
func(t *testing.T, routes, _ *fakeState) {
c := routes.route(t, "anchor.grafana.example").Certificate
if c.Outcome != contract.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" {
t.Fatalf("the failed certificate is not on the bus: %+v", c)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
held := newTable()
held.ledger = newLedger()
routes, unrouted := newFakeState(), newFakeState()
held.ledger.attach("anchor", routes, unrouted)
table, public := routesOf(contributions)
held.set(table, public)
held.ledger.flush()
row.act(t, held)
held.ledger.flush()
row.check(t, routes, unrouted)
})
}
}
func mustFloat(t *testing.T, port string) float64 {
t.Helper()
var n float64
if err := json.Unmarshal([]byte(port), &n); err != nil {
t.Fatal(err)
}
return n
}
modules/route-proxy/cmd/route-proxy/main.go
-975
View File
@@ -1,975 +0,0 @@
// A reverse proxy, in the form the mesh expects one.
//
// **A route is a provision** (novox/hq ADR 0007, 08-connectivity §3; issue 774, research 072 row 34). A
// module that must be reachable declares it requires `route` and contributes the name it wants; the
// proxy provides `route` and serves every consumer whose plan, in the seat mesh-provisioner's state
// `plans`, names it as the route's provider. It is the mirror of a database grant: there the consumer
// supplies a name and receives credentials, here it supplies a target and receives a name.
//
// **It is an example, not part of the controller.** The controller decides and never touches a
// machine. A real deployment runs whatever proxy it likes — the contract is the plan's entry, not
// this program. What lives here is that contract, written as something that runs so it can be read
// rather than described.
//
// **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used
// to say only where to send traffic, so this proxy applied nothing; the four things the ingress it
// replaces actually relies on are now part of the contribution. The set is closed at four, because
// an open middleware surface recreates the thing being replaced and is far harder to narrow later
// than a closed one is to widen.
//
// What it is given, in each plan's entry for the route: every contribution the consumer makes,
// composed by the controller, and where the consumer's machine is on the private network (plans.go).
// Each contribution's values carry the name and port, and optionally:
//
// path the path prefix this rule is scoped to; absent means every path
// priority which rule wins where two match; higher first, and the order is total
// deny refuse the request outright — the shape an incident mitigation needs
// redirect answer with a permanent redirect to this name, keeping the path and query
// auth the *path of a secret* holding `user:hash` lines, never the credential itself
//
// A host may appear more than once, which is what path scoping means: one rule refusing a path
// while another serves everything else on the same name.
//
// **`auth` names a secret and never holds one.** A declaration carrying a credential is refused
// outright rather than served unprotected, and a secret that cannot be read makes the route refuse
// rather than open — a gate that cannot check is not a gate that opens.
//
// It follows the plans on the bus rather than being restarted, for the same reason the provisioner
// does: a route arriving or leaving is an ordinary event and must not drop the connections of every
// other workload.
package main
import (
"bytes"
"context"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"fmt"
"log"
"net"
"net/http"
"net/http/httputil"
"net/netip"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"golang.org/x/crypto/acme"
"golang.org/x/crypto/acme/autocert"
"golang.org/x/crypto/bcrypt"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// Where public certificates come from when nothing says otherwise.
//
// **Staging, deliberately** (novox/hq 04-ISSUES/004). Production issuance is rate-limited per
// domain and per account, the quota does not replenish quickly, and exhausting it removes the
// ability to issue a certificate somebody actually needs. Defaulting to production would leave
// the safe path depending on remembering to opt out of it, on exactly the work most likely to
// iterate — standing up a node, changing how names resolve.
//
// A staging certificate is trusted by no browser, so the mistake announces itself on the first
// request rather than a fortnight later at the rate limit.
const stagingDirectory = "https://acme-staging-v02.api.letsencrypt.org/directory"
// issuer is the ACME directory to ask. The lab points this at its own issuer; a node serving real
// traffic points it at production, and says so explicitly.
func issuer() string {
if named := strings.TrimSpace(os.Getenv("ACME_DIRECTORY")); named != "" {
return named
}
return stagingDirectory
}
// onlyWhatTheMeshSaid refuses to obtain a certificate for a name this proxy was not given.
//
// **The policy that stops a quota from being spent by accident.** Without it, anything that can
// reach port 443 and send a name triggers an issuance attempt for it — so a scan, or one
// misconfigured client, becomes a stream of failed orders against the account's rate limit. What
// this proxy may certify is exactly what the mesh told it to route, which is already the answer
// to what it may serve.
func onlyWhatTheMeshSaid(held *table) autocert.HostPolicy {
return func(_ context.Context, host string) error {
if held.eligibleForACME(host) {
return nil
}
return fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", host)
}
}
// onlyInternalNamesTheMeshSaid is onlyWhatTheMeshSaid's mirror for the internal authority — the
// same quota-spending concern applies even to an authority with no rate limit of its own, because
// an order for a name this proxy does not actually route is a bug worth refusing rather than
// serving.
func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy {
return func(_ context.Context, host string) error {
if held.eligibleForInternalACME(host) {
return nil
}
return fmt.Errorf("no internal-only route for %q in this mesh, so no certificate is asked for", host)
}
}
// contribution is one route a consumer asked for: the module, its machine, where that machine is, the local name of
// its contribution, and the route, composed by the mesh and read with route-proxy's contract (plans.go).
type contribution struct {
From string `json:"from"`
Node string `json:"node"`
// At is where that machine is on the private network. The mesh knows it; a proxy that had to
// build it from the node name would be a naming convention copied into every provider.
At string `json:"at"`
Local string `json:"local,omitempty"`
Route contract.Request `json:"route"`
}
// policy is what a rule does with a request that matched it.
//
// **Decided by the mesh, not here** (novox/hq ADR 0108). A route grant used to hand back a name and
// say nothing about what the name admitted, so this proxy admitted everything. The set is closed at
// four — authentication, refusal, path scoping, redirect — because an open middleware surface
// recreates the thing being replaced and is far harder to narrow later than a closed one is to widen.
type policy struct {
// deny refuses the request outright, whatever it is.
deny bool
// redirectTo answers with a permanent redirect instead of proxying. The request's own path and
// query are carried across, which is what canonicalising one public name onto another means.
redirectTo string
// users is what a request must present, read at load time from the secret the declaration
// *named*. A declaration never carries the credential itself.
users map[string]string
// sealed is set when authentication was declared and the secret could not be read. The rule then
// refuses everything and says why.
//
// **Fail closed.** The alternative — serve the route unauthenticated because the gate is
// missing — turns an unreadable file into a silently public admin surface, which is the exact
// outcome ADR 0108 exists to prevent. A gate that cannot check is not a gate that opens.
sealed string
}
// rule is one way a host may be routed. A host may have several, which is what path scoping means.
type rule struct {
path string // "" matches every path
priority int
policy policy
to *httputil.ReverseProxy
target string
// insecure skips certificate verification when target is reached over https. For a backend
// that terminates TLS with its own certificate this proxy has no reason to trust — Mailu's
// webmail front is the first of these — never for anything reached over plain http, where
// there is nothing to verify in the first place.
insecure bool
// maxRequestBody is the largest body, in bytes, this route carries. Zero is no limit, which is
// what every route gets by saying nothing: this proxy has never limited a body, and a default
// arriving with the field would change every route that never asked for one.
//
// **Configuration, not a policy** (novox/hq ADR 0108 closed that set at four). It belongs beside
// `insecure` for the same reason `insecure` is not a policy: both tune how this proxy carries a
// request to a backend, rather than deciding what the name admits or who may reach it. A
// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
// a proxy's own default refuses them long before the workload is reached.
maxRequestBody int64
// from and node are the consumer that contributed the rule and its machine, which the bus is told (issue 774).
from, node string
}
// table is what the proxy is currently serving, replaced whole whenever the plans change what they give it.
//
// Replaced rather than merged: the plans are the whole truth about who has a route, so merging
// would keep serving a name whose module was unassigned — which is the stale-route fault
// 08-connectivity lists as open, reintroduced one level down.
//
// Keyed by host to an *ordered* list rather than to one target, because two of the four policies
// need a single host routed more than one way: a refusal on a path the ordinary route also matches,
// and a certificate-challenge path on a host that otherwise serves a workload.
type table struct {
mu sync.RWMutex
to map[string][]rule
// public is which routed hosts are eligible for a real certificate — every host reached as a
// route's own `name`, never one reached only as its `internal-name`. A private alias can never
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
public map[string]bool
// inside is where a request must come from to be served a name that is only internal: every
// machine's address on the private network, as the mesh issued it in this proxy's membership
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
inside sources
// ledger is what this proxy says on the bus about what it serves and refuses (issue 774); nil says nothing.
ledger *ledger
}
// sources is who may be served an internal name: the private network's addresses as the mesh
// issued them. The machine itself is always inside — anything on a machine may call anything on it
// (novox/hq ADR 0144) — so loopback needs no entry.
type sources []netip.Prefix
// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does
// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to
// fewer machines than the mesh said, and say nothing.
func sourcesOf(mesh []string) (sources, error) {
var out sources
for _, entry := range mesh {
entry = strings.TrimSpace(entry)
if prefix, err := netip.ParsePrefix(entry); err == nil {
out = append(out, prefix.Masked())
continue
}
addr, err := netip.ParseAddr(entry)
if err != nil {
return nil, fmt.Errorf("%q is not an address on the private network", entry)
}
addr = addr.Unmap()
out = append(out, netip.PrefixFrom(addr, addr.BitLen()))
}
return out, nil
}
// holds says whether a request from this remote address came from the mesh or the machine itself.
//
// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source
// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request
// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a
// mesh address leave by the tunnel, never back to the claimant.
func (s sources) holds(remote string) bool {
host := remote
if h, _, err := net.SplitHostPort(remote); err == nil {
host = h
}
addr, err := netip.ParseAddr(host)
if err != nil {
return false
}
addr = addr.Unmap()
if addr.IsLoopback() {
return true
}
for _, prefix := range s {
if prefix.Contains(addr) {
return true
}
}
return false
}
func (t *table) set(routes map[string][]rule, public map[string]bool) {
made := map[string][]rule{}
for host, rules := range routes {
kept := make([]rule, 0, len(rules))
for _, r := range rules {
// A rule that only refuses or only redirects has nowhere to send anything, and needs
// nowhere: it answers by itself.
if r.policy.deny || r.policy.redirectTo != "" {
kept = append(kept, r)
continue
}
where, err := url.Parse(r.target)
if err != nil {
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
continue
}
r.to = towards(where)
if r.insecure {
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
}
kept = append(kept, r)
}
if len(kept) == 0 {
continue
}
inOrder(kept)
made[host] = kept
}
t.mu.Lock()
t.to = made
t.public = public
t.mu.Unlock()
t.ledger.serving(made, public)
}
// inOrder puts the rules for one host into the order they are matched in, and does so totally.
//
// **Equal priorities must resolve identically every time** (ADR 0108). Sorting only by priority
// leaves rules that share one in whatever order the map produced, so the same declaration would
// serve differently between restarts — a proxy that is not reproducible. Longest path first within a
// priority is also the intuitive reading: the more specific rule wins. The last two keys exist only
// to make the order total.
func inOrder(rules []rule) {
sort.SliceStable(rules, func(i, j int) bool {
a, b := rules[i], rules[j]
if a.priority != b.priority {
return a.priority > b.priority
}
if len(a.path) != len(b.path) {
return len(a.path) > len(b.path)
}
if a.path != b.path {
return a.path < b.path
}
return a.target < b.target
})
}
// find is the rule that answers this request, or nothing if the host is not routed here at all.
func (t *table) find(host, path string) (rule, bool) {
t.mu.RLock()
defer t.mu.RUnlock()
for _, r := range t.to[bareHost(host)] {
if r.path == "" || strings.HasPrefix(path, r.path) {
return r, true
}
}
return rule{}, false
}
// routed says whether this proxy serves the name at all, whatever the path.
//
// Separate from find because certificate issuance is a question about the *name*: a host whose only
// rules are path-scoped is still a name this proxy answers to, and still needs a certificate.
// eligibleForACME says whether this proxy may ask a certificate authority for this name — every
// host reached as a route's own public `name`, never one reached only as its `internal-name`
// alias, which no public CA can ever validate.
func (t *table) eligibleForACME(host string) bool {
t.mu.RLock()
defer t.mu.RUnlock()
bare := bareHost(host)
return len(t.to[bare]) > 0 && t.public[bare]
}
func (t *table) routed(host string) bool {
t.mu.RLock()
defer t.mu.RUnlock()
return len(t.to[bareHost(host)]) > 0
}
// eligibleForInternalACME says whether this proxy may ask its *internal* authority for a
// certificate for this name — every host it routes that is not also a route's public `name`.
//
// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity
// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is
// composed only from `to` and `public`, which routesOf already builds correctly — a host never
// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be
// tracked to tell the two apart.
func (t *table) eligibleForInternalACME(host string) bool {
t.mu.RLock()
defer t.mu.RUnlock()
bare := bareHost(host)
return len(t.to[bare]) > 0 && !t.public[bare]
}
// bareHost is the name without the port, lower-cased.
//
// The port is not part of the name: a request to app.example:8080 is for app.example. Lower-cased
// because a Host header is not case-sensitive, and a route that only answers the spelling in the
// manifest answers half the requests made to it.
func bareHost(host string) string {
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
return strings.ToLower(host)
}
// hiddenFrom says whether this host must look unrouted to a request from this address: it is
// only an internal name, and the request did not come from the private network.
//
// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true**
// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from
// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does.
// Answered exactly as a name that was never routed, so an outsider learns nothing from asking.
func (t *table) hiddenFrom(host, remote string) bool {
if !t.eligibleForInternalACME(host) {
return false
}
t.mu.RLock()
defer t.mu.RUnlock()
return !t.inside.holds(remote)
}
// setInside replaces who the mesh is, as the membership said.
func (t *table) setInside(inside sources) {
t.mu.Lock()
t.inside = inside
t.mu.Unlock()
}
// namesSeenFrom is what this proxy says it serves to a request from this address — every routed
// name, less the internal-only ones when the request came from outside.
func (t *table) namesSeenFrom(remote string) []string {
out := []string{}
for _, name := range t.names() {
if !t.hiddenFrom(name, remote) {
out = append(out, name)
}
}
return out
}
// served is every name served, as an event says it: how far it reaches and the module whose rule serves it first.
func (t *table) served() map[string]contract.Event {
t.mu.RLock()
defer t.mu.RUnlock()
out := make(map[string]contract.Event, len(t.to))
for name, rules := range t.to {
e := contract.Event{Name: name, Reach: contract.ReachInternal}
if t.public[name] {
e.Reach = contract.ReachPublic
}
if len(rules) > 0 {
e.Module, e.Node = rules[0].from, rules[0].node
}
out[name] = e
}
return out
}
func (t *table) names() []string {
t.mu.RLock()
defer t.mu.RUnlock()
out := make([]string, 0, len(t.to))
for name := range t.to {
out = append(out, name)
}
sort.Strings(out)
return out
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, "mesh-route-proxy: "+err.Error())
os.Exit(1)
}
}
func run() error {
listen := strings.TrimSpace(os.Getenv("LISTEN"))
if listen == "" {
listen = ":80"
}
// **The seat's plans are the one source of routes** (issue 774, research 072 row 34). Without the bus this proxy
// knows no route, so it refuses to start rather than serving an empty table and saying nothing.
credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE"))
if credential == "" {
return fmt.Errorf("MESH_BROKER_FILE is not set, so this proxy cannot read the plans that say what it routes")
}
held := newTable()
held.ledger = newLedger()
go held.ledger.loop()
go follow(credential, held)
// TLS is opt-in. A proxy with no `TLS_LISTEN` serves plain HTTP exactly as before — which is
// what an internal-only mesh wants, and what the mesh's own certificate authority already
// covers for names inside it (novox/hq 08-connectivity). This is for names reachable from
// outside, where the authority has to be one the world already trusts.
secure := strings.TrimSpace(os.Getenv("TLS_LISTEN"))
if secure == "" {
return http.ListenAndServe(listen, handler(held))
}
cache := strings.TrimSpace(os.Getenv("ACME_CACHE"))
if cache == "" {
// Refused rather than defaulted. Without somewhere durable to keep them, every restart
// orders new certificates — which works, silently, until the rate limit says it does not.
return fmt.Errorf("TLS_LISTEN is set and ACME_CACHE is not: certificates need somewhere " +
"to persist, or every restart orders them again")
}
publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
onlyWhatTheMeshSaid(held), held)
if err != nil {
return err
}
log.Printf("issuing public certificates from %s, for whatever the mesh routes here", issuer())
// The internal authority is optional: unset means this proxy serves internal-only aliases over
// plain HTTP exactly as it always has, which is the standalone-binary default and a safe one —
// it asks nothing of an authority it was not told about.
var internalManager *autocert.Manager
if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
onlyInternalNamesTheMeshSaid(held), held)
if err != nil {
return fmt.Errorf("internal certificate authority: %w", err)
}
log.Printf("issuing internal certificates from %s, for every internal-only alias this routes",
directory)
}
// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
// must be answered *at the name being certified*, which is why issuance happens on the node
// that is publicly reachable rather than wherever the workload runs.
//
// **autocert's own HTTPHandler does not fall through on the challenge path.** For a token it
// does not hold it answers 404 itself; its fallback only ever sees non-challenge paths — which
// is exactly the predecessor's fault, the edge owning `/.well-known/acme-challenge` outright,
// rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute
// probes each manager and hands a token neither authority recognises to plain routing, which
// is what lets a consumer's own ACME client — Mailu's, certifying its own name for a protocol
// this proxy never proxies — answer its own challenge through an ordinary path-scoped route.
port80 := tokenOrRoute(handler(held), publicManager)
if internalManager != nil {
port80 = tokenOrRoute(handler(held), publicManager, internalManager)
}
go func() {
if err := http.ListenAndServe(listen, port80); err != nil {
log.Printf("plain HTTP stopped: %v", err)
}
}()
tlsConfig := publicManager.TLSConfig()
tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager)
server := &http.Server{
Addr: secure,
Handler: handler(held),
TLSConfig: tlsConfig,
}
return server.ListenAndServeTLS("", "")
}
// tokenOrRoute serves port 80: each manager answers the challenge tokens it is itself holding,
// and a token none of them holds is routed like any other request instead of being 404'd at the
// edge.
//
// autocert gives no way to ask "is this your token?" — its HTTPHandler both answers and refuses —
// so each manager is probed against a buffered writer and its refusal (404 on the challenge path)
// is discarded in favour of the next candidate. The probe is cheap: the handler answers from
// memory, and the path only carries traffic while an issuance is actually running.
func tokenOrRoute(routes http.Handler, managers ...*autocert.Manager) http.Handler {
const challengePrefix = "/.well-known/acme-challenge/"
// Non-challenge paths never reach a manager at all; autocert's tryHTTP01 switch still has to
// be armed, which HTTPHandler is the only exported way to do.
probes := make([]http.Handler, len(managers))
for i, m := range managers {
probes[i] = m.HTTPHandler(routes)
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, challengePrefix) {
routes.ServeHTTP(w, r)
return
}
for _, probe := range probes {
buffered := &probedResponse{header: make(http.Header)}
probe.ServeHTTP(buffered, r)
// Two shapes of "not mine": 404, a token this manager is not holding — and 403, a
// name its host policy would never certify at all (autocert checks the policy before
// the token, so the internal authority answers 403 for every public name).
if buffered.status == http.StatusNotFound || buffered.status == http.StatusForbidden {
continue
}
buffered.replayTo(w)
return
}
routes.ServeHTTP(w, r) // no authority holds it: the workload behind a routed path may
})
}
// probedResponse buffers one handler's answer so a refusal can be discarded unseen.
type probedResponse struct {
header http.Header
status int
body bytes.Buffer
}
func (p *probedResponse) Header() http.Header { return p.header }
func (p *probedResponse) WriteHeader(status int) {
if p.status == 0 {
p.status = status
}
}
func (p *probedResponse) Write(b []byte) (int, error) {
if p.status == 0 {
p.status = http.StatusOK
}
return p.body.Write(b)
}
func (p *probedResponse) replayTo(w http.ResponseWriter) {
for k, vs := range p.header {
for _, v := range vs {
w.Header().Add(k, v)
}
}
status := p.status
if status == 0 {
status = http.StatusOK
}
w.WriteHeader(status)
_, _ = w.Write(p.body.Bytes())
}
// newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and
// which names it may be asked to certify.
//
// **Trusting an authority names a file rather than skipping verification.** An issuer that is not
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
func newManager(cache, directory, bundle string, policy autocert.HostPolicy, held *table) (*autocert.Manager, error) {
client := &acme.Client{DirectoryURL: directory}
var root []byte
if bundle != "" {
read, err := os.ReadFile(bundle)
if err != nil {
return nil, fmt.Errorf("the CA bundle names %s and it cannot be read: %w", bundle, err)
}
root = read
// An empty bundle means the issuer's root is already in the system trust store — a public
// authority whose root ships with the OS, pointed at by a provider that serves an empty
// root (novox/hq ADR 0066). The mesh always writes the bundle file, so it exists and holds
// nothing; that is the signal to fall back to the system roots, the same as if nothing had
// named a bundle at all. A file that holds bytes but no certificate is still a
// misconfiguration and is refused, because there the operator meant to trust something.
if strings.TrimSpace(string(root)) != "" {
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(root) {
return nil, fmt.Errorf("%s holds no certificate this can trust", bundle)
}
client.HTTPClient = &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}},
}
}
}
// Where this authority's account and certificates are kept. Per authority, not per proxy — see
// forThisAuthority, which is what makes a re-initialised CA heal itself, and what lets the
// public and internal authorities share one ACME_CACHE without colliding: they hash to
// different names because their directories differ.
mine := forThisAuthority(cache, directory, root)
return &autocert.Manager{
Cache: recordingCache{Cache: autocert.DirCache(mine), held: held},
Prompt: autocert.AcceptTOS,
HostPolicy: policy,
Client: client,
}, nil
}
// forThisAuthority is where one ACME authority's account and certificates are kept.
//
// **A cached ACME account belongs to the authority that issued it, and nothing in the cache says
// so** (novox/hq ADR 0066). autocert keeps its account key at one fixed name — `acme_account+key` —
// in whatever directory it is given, and reuses it for ever. That is right while the authority stays
// the same and silently wrong the moment it does not: an internal CA that is re-initialised is a new
// authority with a new root, it has never heard of the account in the cache, and every attempt to
// use it is rejected. autocert has no path back from that. Nothing is retried, nothing is
// re-registered, no order ever reaches the CA — issuance simply stops, with no error anybody sees,
// until a person deletes the directory by hand and finds out that was the answer.
//
// So the directory is named after the authority instead of being shared by all of them. The name is
// a digest of the two things that identify one: the directory URL, and the root this proxy was told
// to verify it with. Re-initialising the CA produces a new root; the mesh delivers it as a changed
// bundle; this proxy restarts on that file and lands in a directory with no account in it, so
// autocert registers afresh and orders again. **The healing is that the question "is this account
// still valid" never has to be asked** — an account is only ever found where it is still valid.
//
// It also fixes a latent one of the same shape: pointing ACME_DIRECTORY at production after testing
// against staging reused the staging account, because the cache had no idea they were different.
//
// The old directories stay on disk, unused. Left rather than deleted: they are the only copy of
// certificates that may still be valid, and this program is not the thing that should decide a
// certificate is finished with.
func forThisAuthority(cache, directory string, root []byte) string {
sum := sha256.Sum256([]byte(directory + "\x00" + string(bytes.TrimSpace(root))))
return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
}
// certificateFor picks the certificate a handshake is answered with.
//
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than only when
// an order is placed, since a cached certificate is served here on every request and never goes
// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the
// private network asking for a name that is only internal: the certificate would name it.
func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error),
internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error)
if internalManager != nil {
fromInternal = internalManager.TLSConfig().GetCertificate
}
pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
held.ledger.refused(bareHost(hello.ServerName), true, true)
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
hello.ServerName)
}
if fromInternal != nil {
return fromInternal(hello)
}
}
return fromPublic(hello)
}
// What the bus is told of a handshake (issue 774): a routed name's certificate, or why it has none; a name not
// routed here, which a browser asking for a site the mesh forgot to route meets before any request is made.
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
cert, err := pick(hello)
name := bareHost(hello.ServerName)
switch {
case name == "":
case !held.routed(name):
held.ledger.refused(name, false, true)
case err != nil:
if !strings.HasPrefix(err.Error(), "no public route for") {
held.ledger.certificateFailed(name, err)
}
case cert != nil:
leaf := cert.Leaf
if leaf == nil && len(cert.Certificate) > 0 {
leaf, _ = x509.ParseCertificate(cert.Certificate[0])
}
held.ledger.handedOut(name, leaf)
}
return cert, err
}
}
// newTable is an empty routing table.
func newTable() *table {
return &table{to: map[string][]rule{}}
}
// handler is the proxy itself, separated so it can be driven by a test without a listener.
func handler(held *table) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hidden := held.hiddenFrom(r.Host, r.RemoteAddr)
matched, known := held.find(r.Host, r.URL.Path)
if hidden || !known {
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
// are different things, and a proxy that says only "not found" makes an operator go
// and read the mesh to tell them apart. What it is serving is the answer to both.
//
// And since a host may now be routed only on some paths, those are a third thing:
// saying "no route for this name" while listing that very name as served is a
// contradiction an operator would have to disbelieve the proxy to get past.
// **Said in the log as well as to the client.** A name this mesh does not serve, asked
// for from outside, is what a scanner does, and the machine's intrusion prevention reads
// this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the
// jail's filter expects it.
log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
if hidden || !held.routed(r.Host) {
held.ledger.refused(bareHost(r.Host), hidden, false)
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound)
if !hidden && held.routed(r.Host) {
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
bareHost(r.Host), r.URL.Path)
return
}
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", "))
return
}
switch {
case matched.policy.sealed != "":
// Declared a gate, cannot check it. Refused, and says why — an operator reading this
// learns the secret is missing, rather than wondering why a protected name is 503.
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusServiceUnavailable)
fmt.Fprintf(w, "this route requires authentication and its credentials cannot be read: %s\n",
matched.policy.sealed)
return
case matched.policy.deny:
http.Error(w, "this path is not served to you", http.StatusForbidden)
return
case matched.policy.redirectTo != "":
http.Redirect(w, r, canonical(matched.policy.redirectTo, r.URL), http.StatusMovedPermanently)
return
case len(matched.policy.users) > 0 && !allowed(matched.policy.users, r):
// The realm is the name asked for, so a browser's prompt says which route it is for.
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q, charset=\"UTF-8\"", bareHost(r.Host)))
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
if matched.maxRequestBody > 0 {
// Refused on the declared length where there is one, so an upload that cannot succeed
// is answered before it is carried; and capped while reading for a chunked body, which
// declares no length at all. Without the second, a limit is advice.
if r.ContentLength > matched.maxRequestBody {
http.Error(w, fmt.Sprintf("request body too large for this route: %d bytes is the most it carries",
matched.maxRequestBody), http.StatusRequestEntityTooLarge)
return
}
r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
}
// The upstream's answer is what the bus is told of this name (issue 774): its status, or why there was none.
answer := &answerRecorder{ResponseWriter: w}
matched.to.ServeHTTP(answer, r)
status := answer.status
if status == 0 && answer.err == nil && r.Header.Get("Upgrade") != "" {
status = http.StatusSwitchingProtocols
}
held.ledger.answered(bareHost(r.Host), r.URL.Path, status, answer.err)
})
}
// canonical is where a redirect sends this request.
//
// The declaration names the destination *name*; the request keeps its own path and query. That is
// what canonicalising one public name onto another means — a link to a page under the old name has
// to arrive at the same page under the new one, or the redirect silently loses every deep link.
func canonical(to string, from *url.URL) string {
where, err := url.Parse(to)
if err != nil {
return to
}
if where.Path == "" || where.Path == "/" {
where.Path = from.Path
}
if where.RawQuery == "" {
where.RawQuery = from.RawQuery
}
return where.String()
}
// allowed says whether the request presented credentials this route accepts.
//
// **Every path costs one bcrypt comparison**, including an unknown user, which is why the miss
// compares against a fixed hash rather than returning early. Returning early would make an unknown
// user measurably faster than a known one with a wrong password, and that difference is a way to
// enumerate the users of a route from outside it.
func allowed(users map[string]string, r *http.Request) bool {
// A hash of nothing anybody knows. Its only job is to cost what a real comparison costs.
const absent = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"
user, password, ok := r.BasicAuth()
if !ok {
return false
}
want, known := users[user]
if !known {
want = absent
}
if err := bcrypt.CompareHashAndPassword([]byte(want), []byte(password)); err != nil {
return false
}
// `known` is checked after the comparison, not instead of it, so the timing is the same either
// way. subtle.ConstantTimeByteEq keeps the branch from being the thing that differs.
return subtle.ConstantTimeByteEq(boolByte(known), 1) == 1
}
func boolByte(b bool) byte {
if b {
return 1
}
return 0
}
// routesOf turns the routes the plans give into host → the rules for that host, and which of those hosts is a public
// name — the second is `name`, ACME-eligible; a host reached only through `internal-name` never appears there.
//
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches decides which names the
// mesh composes, so an endpoint that reaches only the private network arrives with an `internal-name` and no `name`.
// That is a whole route, not a malformed one: it is served under its internal name and certified by the internal
// authority. Only a route with neither name has nothing to be served under (novox/hq issue 191).
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
out := map[string][]rule{}
public := map[string]bool{}
for _, c := range contributions {
// The contract admitted the route whole (contract.Request.Validate): it names a host, a rule that forwards
// gives a port, its scheme is http or https, its body limit is whole and positive, and its auth names a
// secret's file rather than holding a credential (novox/hq ADR 0108).
r := c.Route
name := strings.TrimSpace(r.Name)
internal := strings.TrimSpace(r.InternalName)
made := rule{path: r.Prefix(), priority: r.Priority}
made.policy.deny, made.policy.redirectTo = r.Deny, r.Redirect
if named := strings.TrimSpace(r.Auth); named != "" {
users, err := usersFrom(named)
if err != nil {
// Fail closed: the rule is kept so the name stays routed and answers, and it
// answers by refusing. Dropping it instead would make the name 404 and read as a
// withdrawn route rather than an unreadable secret.
made.policy.sealed = err.Error()
}
made.policy.users = users
}
// Only a rule that actually proxies needs somewhere to send the request.
if r.Forwards() {
// Where the mesh says that machine is. Empty means it is this one — a workload beside
// the proxy is ordinary, and reaching it over loopback is both correct and the only
// thing that works when there is no private network.
at := c.At
if at == "" {
at = "127.0.0.1"
}
// http unless the contribution says otherwise. A backend that terminates its own TLS
// with a certificate this proxy has no reason to trust — Mailu's webmail front is the
// first of these — is the reason `insecure` exists, and it stays the exception.
made.insecure = r.Insecure
if r.MaxRequestBody != nil {
made.maxRequestBody = *r.MaxRequestBody
}
made.target = fmt.Sprintf("%s://%s:%d", r.SchemeOrDefault(), at, r.Port)
}
made.from, made.node = c.From, c.Node
if name != "" {
host := strings.ToLower(name)
out[host] = append(out[host], made)
public[host] = true
}
// The internal-network name, the same rule under a second host — a predecessor proxy
// answered both for one route, as a convenience (reaching a service over the VPN without a
// public TLS round trip), not as an access boundary; composing it here restores exactly
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
// 0056's internalDomain half). And the only name, when the endpoint reaches no further than
// the private network.
if internal != "" {
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
}
}
return out, public
}
// usersFrom reads the credentials the mesh mounted, in the one format every htpasswd already is.
func usersFrom(path string) (map[string]string, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("cannot read the secret named for this route: %w", err)
}
users := map[string]string{}
for _, line := range strings.Split(string(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
user, hash, ok := strings.Cut(line, ":")
if !ok || user == "" || hash == "" {
continue
}
users[user] = hash
}
if len(users) == 0 {
return nil, fmt.Errorf("the secret named for this route holds no usable credentials")
}
return users, nil
}
// towards proxies to one backend and tells it what the client asked: **X-Forwarded-Proto, -Host and
// -For**, set from the request this proxy received. A backend that builds its own addresses — a forge
// writing its clone URL into a page, a login redirect — otherwise sees the plain HTTP hop from this
// proxy and writes `http://`, though every client reached it over TLS: Go refused the forge's module
// path for exactly that on 2026-10-03, its import tag naming an http clone URL.
// The standard library's NewSingleHostReverseProxy sets only X-Forwarded-For.
func towards(where *url.URL) *httputil.ReverseProxy {
return &httputil.ReverseProxy{Rewrite: func(pr *httputil.ProxyRequest) {
pr.SetURL(where)
pr.Out.Host = pr.In.Host
pr.SetXForwarded()
}, ErrorHandler: proxyFailed}
}
modules/route-proxy/cmd/route-proxy/plans.go
-147
View File
@@ -1,147 +0,0 @@
package main
// The routes this proxy serves, read from the seat mesh-provisioner's plans and nothing else (issue 774; research 072
// row 34, the operator on 2026-10-31: "there is only 1 path, the other is simply deprecated/invalid").
//
// **A route is a provision, in route-proxy's own contract.** A module that must be reachable requires `route`; the
// controller composes its plan on its node, and the plan's entry for the route names this proxy as its provider and
// carries what the module contributes, composed (its names joined to the node's domains, the machine port its
// placement assigned its endpoint, never the software's own), and where the module's machine is on the private
// network, as one opaque payload the seat never reads (mesh-sdk catalogue.PlanProvision.Contributed). The proxy decodes each contribution with its contract (contract.DecodeRequest)
// and serves every entry naming it. There is no file and no second source: a route the plans do not name is not
// served, and a route the contract refuses is said refused (contract.EventRefused), never served half.
import (
"fmt"
"log"
"sort"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// planned is the routes the seat's plans give this proxy on its node, by plan key, kept so a plan that cannot be read
// leaves what it gave served (a bind missed would read as a route withdrawn).
type planned struct {
node string
// said is where a refusal is told: the bus's events, nothing in a test that does not ask.
said func(eventType string, e contract.Event)
mu sync.Mutex
by map[string][]contribution
}
func newPlanned(node string, said func(string, contract.Event)) *planned {
if said == nil {
said = func(string, contract.Event) {}
}
return &planned{node: node, said: said, by: map[string][]contribution{}}
}
// take is one change of a plan's key: its value, or nil when the key was deleted. It answers whether what is served
// changed, and says on the log and the bus what it could not serve.
func (p *planned) take(key string, value []byte) bool {
p.mu.Lock()
defer p.mu.Unlock()
if value == nil {
_, had := p.by[key]
delete(p.by, key)
return had
}
gave, refused, err := routesOfPlan(p.node, key, value)
if err != nil {
log.Printf("the plan %s is not readable, so what it routes is kept as it was: %v", key, err)
return false
}
for _, r := range refused {
log.Printf("the plan %s gives %s on %s a route this proxy does not serve: %s", key, r.Module, r.Node, r.Why)
p.said(contract.EventRefused, r)
}
if len(gave) == 0 {
_, had := p.by[key]
delete(p.by, key)
return had
}
p.by[key] = gave
return true
}
// contributions is every route the plans give this proxy, in the order of their plans' keys.
func (p *planned) contributions() []contribution {
p.mu.Lock()
defer p.mu.Unlock()
keys := make([]string, 0, len(p.by))
for k := range p.by {
keys = append(keys, k)
}
sort.Strings(keys)
var out []contribution
for _, k := range keys {
out = append(out, p.by[k]...)
}
return out
}
// routesOfPlan is the routes one plan gives the proxy on node: each contribution the plan's entries naming this proxy
// on node as the route's provider carry, decoded with the contract, reached on its own machine over loopback and on
// another at the address the plan says. A plan of another module, or one routing elsewhere, gives none. Refused is
// what the plan names and the proxy does not serve, each with why: an entry carrying nothing to serve, a payload or a
// route the contract does not read, or a consumer elsewhere with no address on the private network.
func routesOfPlan(node, key string, value []byte) ([]contribution, []contract.Event, error) {
plan, err := provisions.DecodePlan(value)
if err != nil {
return nil, nil, err
}
if plan.Node+"."+plan.Module != key {
return nil, nil, fmt.Errorf("it is the plan of %s.%s, under the key %s", plan.Node, plan.Module, key)
}
var out []contribution
var refused []contract.Event
refuse := func(local, why string) {
refused = append(refused, contract.Event{Proxy: node, Module: plan.Module, Node: plan.Node, Local: local, Why: why})
}
named, carried := false, false
for _, e := range plan.Plan.Provisions {
if !routesHere(node, e) {
continue
}
named = true
// What the module contributes is on the requirement's first entry alone, however many local names it has.
if len(e.Contributed) == 0 {
continue
}
carried = true
given, err := catalogue.DecodeContributed(e.Contributed)
if err != nil {
refuse("", err.Error())
continue
}
at := ""
if plan.Node != node {
if given.At == "" {
refuse("", fmt.Sprintf("it runs on %s and its plan says no address on the private network for it", plan.Node))
continue
}
at = given.At
}
for _, one := range given.Each {
route, err := contract.DecodeRequest(one.Values)
if err != nil {
refuse(one.Local, err.Error())
continue
}
out = append(out, contribution{From: plan.Module, Node: plan.Node, At: at, Local: one.Local, Route: route})
}
}
if named && !carried {
refuse("", "its plan names this proxy for its route and carries nothing to serve")
}
return out, refused, nil
}
// routesHere is whether a plan's entry is a route this proxy on node provides.
func routesHere(node string, e catalogue.PlanProvision) bool {
return e.Provision == contract.Provision && e.Provider == node && e.ProviderModule == contract.Module
}
modules/route-proxy/cmd/route-proxy/policy_test.go
-265
View File
@@ -1,265 +0,0 @@
package main
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"golang.org/x/crypto/bcrypt"
)
// What a route carries about the requests arriving at it — novox/hq ADR 0108.
//
// Each test here is one of the four capabilities that record closed the set at, plus the negative
// case it promised would be refused. The negative case is the one that rots quietly: nothing fails
// if it stops working, so nothing tells you it has.
// served starts a workload and gives back the host and port the mesh would have recorded for it.
func served(t *testing.T, body string) (string, int) {
t.Helper()
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(body))
}))
t.Cleanup(workload.Close)
host, port, _ := strings.Cut(strings.TrimPrefix(workload.URL, "http://"), ":")
n, err := strconv.Atoi(port)
if err != nil {
t.Fatal(err)
}
return host, n
}
// ask makes one request through the proxy for a given name and path, without following redirects.
func ask(t *testing.T, proxy, name, path string, auth [2]string) *http.Response {
t.Helper()
req, err := http.NewRequest(http.MethodGet, proxy+path, nil)
if err != nil {
t.Fatal(err)
}
req.Host = name
if auth[0] != "" {
req.SetBasicAuth(auth[0], auth[1])
}
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}}
answer, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = answer.Body.Close() })
return answer
}
func proxyFor(t *testing.T, routesJSON string) string {
t.Helper()
routes, public, err := routesFrom(routesJSON)
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, public)
server := httptest.NewServer(handler(held))
t.Cleanup(server.Close)
return server.URL
}
// A refusal on a path shadows the ordinary route for that path and leaves every other path alone.
//
// **This is why path scoping is a prerequisite and not a sibling capability.** The rule being
// reproduced matches a path on a host that is already routed to a workload, so a table mapping a
// host to one target cannot express it at all — no amount of authentication or source filtering
// would have helped.
func TestARefusedPathShadowsTheRouteAndLeavesTheRestServed(t *testing.T) {
at, port := served(t, "the workload")
proxy := proxyFor(t, `{"given":[
{"from":"forge","node":"anchor","at":"`+at+`","values":{"name":"forge.example","port":`+strconv.Itoa(port)+`}},
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","priority":100,"deny":true}}
]}`)
if got := ask(t, proxy, "forge.example", "/api/internal/hook", [2]string{}).StatusCode; got != http.StatusForbidden {
t.Fatalf("the refused path answered %d, so the block that was put in front of it during an "+
"incident is not in front of it any more", got)
}
if got := ask(t, proxy, "forge.example", "/", [2]string{}).StatusCode; got != http.StatusOK {
t.Fatalf("refusing one path took the whole route with it: %d", got)
}
}
// A redirect answers with the redirect, and the request keeps its own path and query.
//
// Losing the path would turn canonicalising one name onto another into "every deep link now lands
// on the front page", which is the kind of breakage that produces no error anywhere.
func TestARedirectKeepsThePathAndQuery(t *testing.T) {
proxy := proxyFor(t, `{"given":[
{"from":"site","node":"anchor","values":{"name":"www.example","redirect":"https://example/"}}
]}`)
answer := ask(t, proxy, "www.example", "/deep/page?ref=1", [2]string{})
if answer.StatusCode != http.StatusMovedPermanently {
t.Fatalf("a declared redirect answered %d", answer.StatusCode)
}
where := answer.Header.Get("Location")
if !strings.Contains(where, "/deep/page") || !strings.Contains(where, "ref=1") {
t.Fatalf("the redirect dropped the path or the query: %q", where)
}
}
// Authentication refuses a request with no credentials, admits one with the right ones, and refuses
// the wrong ones — with the credentials read from the secret the declaration *named*.
func TestAuthenticationAdmitsOnlyWhatTheSecretSays(t *testing.T) {
at, port := served(t, "the console")
hash, err := bcrypt.GenerateFromPassword([]byte("correct horse"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
secret := filepath.Join(t.TempDir(), "console-auth")
if err := os.WriteFile(secret, []byte("# a comment\nadmin:"+string(hash)+"\n"), 0o600); err != nil {
t.Fatal(err)
}
proxy := proxyFor(t, `{"given":[
{"from":"console","node":"anchor","at":"`+at+`","values":{"name":"console.example","port":`+strconv.Itoa(port)+`,"auth":"`+secret+`"}}
]}`)
if got := ask(t, proxy, "console.example", "/", [2]string{}).StatusCode; got != http.StatusUnauthorized {
t.Fatalf("an admin surface with no login of its own answered %d without credentials", got)
}
if got := ask(t, proxy, "console.example", "/", [2]string{"admin", "wrong"}).StatusCode; got != http.StatusUnauthorized {
t.Fatalf("the wrong password answered %d", got)
}
if got := ask(t, proxy, "console.example", "/", [2]string{"admin", "correct horse"}).StatusCode; got != http.StatusOK {
t.Fatalf("the right password answered %d", got)
}
}
// The negative case ADR 0108 promised would be refused: a credential in the declaration.
//
// **Refused whole, not tolerated and not served unprotected.** A hash carried in a declaration was
// the rejected option; nothing in the running system should quietly accept it later, because the
// precedent is far easier to set than to withdraw. If this test is deleted the option returns and
// nothing else notices.
func TestACredentialInTheDeclarationIsRefusedRatherThanServed(t *testing.T) {
inline := []string{
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"admin:$2a$10$abcdefghijklmnopqrstuv"}}]}`,
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"$2a$10$abcdefghijklmnopqrstuv"}}]}`,
}
for _, body := range inline {
routes, _, err := routesFrom(body)
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 {
t.Fatalf("a declaration carrying a credential was served anyway: %v", routes)
}
}
}
// Authentication declared, secret unreadable: the route refuses. It does not serve unprotected.
//
// **Fail closed.** The alternative turns a missing file into a silently public admin surface, which
// is the outcome the whole record exists to prevent. It answers rather than 404s, so an operator
// sees "cannot read the credentials" instead of concluding the route was withdrawn.
func TestAnUnreadableSecretFailsClosed(t *testing.T) {
at, port := served(t, "the console")
missing := filepath.Join(t.TempDir(), "not-mounted")
proxy := proxyFor(t, `{"given":[
{"from":"console","node":"anchor","at":"`+at+`","values":{"name":"console.example","port":`+strconv.Itoa(port)+`,"auth":"`+missing+`"}}
]}`)
answer := ask(t, proxy, "console.example", "/", [2]string{})
if answer.StatusCode == http.StatusOK {
t.Fatal("a route whose credentials could not be read served the workload unprotected")
}
if answer.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("expected the route to say it cannot check, got %d", answer.StatusCode)
}
}
// Equal priorities resolve the same way every time, so the same declaration serves the same way
// after a restart.
//
// Sorting only by priority leaves rules that share one in whatever order the map produced. The
// proxy would still work, and would work differently between restarts — which is the hardest kind
// of fault to believe when it is reported.
func TestRulesThatShareAPriorityAreStillTotallyOrdered(t *testing.T) {
first := []rule{
{path: "/a", priority: 10, target: "http://x:1"},
{path: "/bb", priority: 10, target: "http://y:2"},
{path: "", priority: 10, target: "http://z:3"},
}
second := []rule{
{path: "", priority: 10, target: "http://z:3"},
{path: "/bb", priority: 10, target: "http://y:2"},
{path: "/a", priority: 10, target: "http://x:1"},
}
inOrder(first)
inOrder(second)
for i := range first {
if first[i].path != second[i].path || first[i].target != second[i].target {
t.Fatalf("two orderings of the same rules disagree at %d: %q vs %q",
i, first[i].path, second[i].path)
}
}
// And the more specific rule is matched first, which is the intuitive reading.
if first[0].path != "/bb" {
t.Fatalf("the longest path is not matched first: %q", first[0].path)
}
}
// Priority decides before path length does, so a rule can be made to win regardless of specificity.
func TestPriorityOutranksPathLength(t *testing.T) {
rules := []rule{
{path: "/very/long/path", priority: 1, target: "http://x:1"},
{path: "", priority: 100, target: "http://y:2"},
}
inOrder(rules)
if rules[0].priority != 100 {
t.Fatalf("a higher priority did not win: %+v", rules[0])
}
}
// A priority above a port number survives, because a priority is an ordering and not a port.
//
// **Found by review, and it was load-bearing.** Priority was first read with the port reader, which
// caps at 65535 — so a rule declared above that silently became priority 0 and stopped shadowing the
// route it exists to shadow. The one real rule this has to reproduce is declared at 100000, so the
// capability would have shipped looking complete and doing nothing.
func TestAPriorityAboveAPortNumberSurvives(t *testing.T) {
at, port := served(t, "the workload")
proxy := proxyFor(t, `{"given":[
{"from":"forge","node":"anchor","at":"`+at+`","values":{"name":"forge.example","port":`+strconv.Itoa(port)+`}},
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","priority":100000,"deny":true}}
]}`)
if got := ask(t, proxy, "forge.example", "/api/internal/hook", [2]string{}).StatusCode; got != http.StatusForbidden {
t.Fatalf("a rule declared at priority 100000 answered %d instead of refusing", got)
}
}
// A host routed only on some paths says so, rather than claiming the name is not served here.
//
// Saying "no route for this name" while listing that very name as served is a contradiction an
// operator has to disbelieve the proxy to get past — and path scoping makes it reachable, because a
// host can now have rules that none of this request's paths match.
func TestAHostRoutedOnlyOnSomePathsSaysSo(t *testing.T) {
proxy := proxyFor(t, `{"given":[
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","deny":true}}
]}`)
answer := ask(t, proxy, "forge.example", "/elsewhere", [2]string{})
if answer.StatusCode != http.StatusNotFound {
t.Fatalf("an uncovered path answered %d", answer.StatusCode)
}
body := make([]byte, 256)
n, _ := answer.Body.Read(body)
said := string(body[:n])
if !strings.Contains(said, "is served here") || !strings.Contains(said, "/elsewhere") {
t.Fatalf("the refusal does not distinguish an uncovered path from an unserved name: %q", said)
}
}
modules/route-proxy/cmd/route-proxy/reach_test.go
-331
View File
@@ -1,331 +0,0 @@
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"slices"
"sort"
"strings"
"testing"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// behind is a workload the proxy can send to, and a table routing one public name and one
// internal-only name to it, with the mesh's machines as the membership would issue them.
func behind(t *testing.T, mesh ...string) *table {
t.Helper()
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "the workload")
}))
t.Cleanup(workload.Close)
at, _ := url.Parse(workload.URL)
host, port, _ := net.SplitHostPort(at.Host)
routes, public, err := routesFrom(fmt.Sprintf(`{"given":[
{"from":"app","node":"anchor","at":%q,
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}},
{"from":"admin","node":"anchor","at":%q,
"values":{"internal-name":"admin.anchor.internal","port":%s}}
]}`, host, port, host, port))
if err != nil {
t.Fatal(err)
}
held := newTable()
inside, err := sourcesOf(mesh)
if err != nil {
t.Fatal(err)
}
held.setInside(inside)
held.set(routes, public)
return held
}
// askFrom is what the proxy answers a request for host coming from remote.
func askFrom(held *table, host, remote string) (int, string) {
r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil)
r.RemoteAddr = remote
w := httptest.NewRecorder()
handler(held).ServeHTTP(w, r)
return w.Code, w.Body.String()
}
// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR
// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name
// being internal kept nobody out: a request from the internet only had to carry it.
func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK ||
body != "the workload" {
t.Errorf("a request from the private network was not served: %d %q", code, body)
}
if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK {
t.Errorf("a request from the machine itself was not served: %d %q", code, body)
}
code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000")
if code != http.StatusNotFound {
t.Fatalf("a request from outside the private network reached an internal-only name: %d %q",
code, body)
}
// Answered as a name never routed, and the list of what is served does not name it either —
// otherwise the refusal would tell an outsider exactly what to ask for from inside.
if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") {
t.Errorf("the refusal names the internal-only route to an outsider: %q", body)
}
if !strings.Contains(body, "app.example") {
t.Errorf("the refusal stopped listing the public names: %q", body)
}
}
// The internal name of a route that also has a public one is internal too: served inside, and to
// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a
// name stays one thing whichever route it came from.
func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK {
t.Errorf("the internal alias stopped answering the private network: %d %q", code, body)
}
if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound {
t.Errorf("the internal alias was served to an outsider: %d", code)
}
if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK {
t.Errorf("the public name was refused to an outsider: %d", code)
}
}
// Before a membership has said who the mesh is, only the machine itself is inside — refused to
// everyone else, never served to everyone.
func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) {
held := behind(t)
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound {
t.Errorf("an internal-only name was served with no private network said: %d", code)
}
if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK {
t.Errorf("an internal-only name was refused to the machine itself: %d", code)
}
}
type from struct {
net.Conn
remote net.Addr
}
func (c from) RemoteAddr() net.Addr { return c.remote }
// The handshake refuses an internal-only name to an outsider too: the certificate would name it,
// and serving it would answer the question the routing refuses to.
func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
served := &tls.Certificate{}
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil)
hello := func(name, remote string) *tls.ClientHelloInfo {
addr, _ := net.ResolveTCPAddr("tcp", remote)
return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}}
}
if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil {
t.Error("an outsider was handed a certificate for an internal-only name")
}
if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served {
t.Errorf("a client on the private network was refused: %v", err)
}
if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served {
t.Errorf("a public name was refused to an outsider: %v", err)
}
}
// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is
// refused rather than skipped, so a typo never quietly narrows or widens who is inside.
func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil {
t.Error("an entry that is not an address was accepted")
}
inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"})
if err != nil {
t.Fatal(err)
}
for remote, want := range map[string]bool{
"10.10.0.1:1": true,
"[::ffff:10.10.0.1]:1": true,
"[fd00::1]:1": true,
"10.20.0.200:1": true,
"10.10.0.2:1": false,
"192.168.1.10:1": false,
"not-an-address": false,
} {
if inside.holds(remote) != want {
t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want)
}
}
}
// fakePlans is the seat's plans as a watch hands them: every current key, then each change after Watch returned.
type fakePlans struct {
current map[string]string
handle func(stdio.StateChange) error
}
func (f *fakePlans) Watch(_ string, handle func(stdio.StateChange) error) error {
f.handle = handle
for k, v := range f.current {
if err := handle(stdio.StateChange{Key: k, Op: "put", Value: json.RawMessage(v), Current: true}); err != nil {
return err
}
}
return nil
}
// plan is a plan of the seat mesh-provisioner for module on node, its entries as given.
func plan(node, module, entries string) string {
return fmt.Sprintf(`{"v":1,"node":%q,"module":%q,"plan":{"templates":{"source":"","digest":""},"files":null,`+
`"provisions":[%s]}}`, node, module, entries)
}
// contributed is a plan entry's opaque payload: the consumer's machine at at, and each contribution by local name.
func contributed(at string, each ...string) string {
var parts []string
for i := 0; i+1 < len(each); i += 2 {
parts = append(parts, fmt.Sprintf(`{"local":%q,"values":%s}`, each[i], each[i+1]))
}
return fmt.Sprintf(`"contributed":{"v":1,"at":%q,"each":[%s]}`, at, strings.Join(parts, ","))
}
// What the plans give is what is served, and nothing else (issue 774, research 072 row 34): each entry naming this
// proxy on its node as the route's provider, every contribution of it read with route-proxy's contract, reached over
// loopback on this machine and at the plan's address on another; an entry routing elsewhere, carrying nothing, naming
// no address or a route the contract refuses is not served, and is said refused. A change is served as it comes and
// said served or withdrawn, a plan that cannot be read keeps what it gave, and a deleted one withdraws it. The
// placement says only who is inside: an internal-only name is served to the machines it decides addresses for.
func TestThePlansAreServedAndThePlacementSaysWhoIsInside(t *testing.T) {
const route = `{"provision":"route","provider":"anchor","provider-module":"route-proxy",`
gitea := route + contributed("10.10.0.7",
"web", `{"name":"git.example","internal-name":"git.anchor.internal","port":3000}`,
"api", `{"internal-name":"git-api.anchor.internal","port":3001}`,
"refused", `{"name":"bad.example","port":3002,"host":"x"}`) + `}`
plans := &fakePlans{current: map[string]string{
"laptop.gitea": plan("laptop", "gitea", gitea+`,{"provision":"postgres-database","provider":"anchor",`+
`"provider-module":"postgres",`+contributed("10.10.0.7", "", `{"name":"gitea"}`)+`}`),
"anchor.blog": plan("anchor", "blog", route+contributed("10.10.0.1", "", `{"name":"blog.example","port":2368}`)+`}`),
"anchor.other": plan("anchor", "other", `{"provision":"route","provider":"edge","provider-module":"route-proxy",`+
contributed("", "", `{"name":"other.example","port":80}`)+`}`),
"anchor.older": plan("anchor", "older", route+`"local":""}`),
"laptop.lost": plan("laptop", "lost", route+contributed("", "", `{"name":"lost.example","port":80}`)+`}`),
}}
held := newTable()
var said []string
say := func(kind string, e contract.Event) {
said = append(said, kind+" "+e.Proxy+" "+e.Name+" "+e.Module+" "+e.Local+" "+e.Why)
}
if err := followPlans(plans, newPlanned("anchor", say), held, say); err != nil {
t.Fatal(err)
}
served := func() map[string]string {
out := map[string]string{}
held.mu.RLock()
defer held.mu.RUnlock()
for host, rules := range held.to {
out[host] = rules[0].target
}
return out
}
want := map[string]string{"git.example": "http://10.10.0.7:3000", "git.anchor.internal": "http://10.10.0.7:3000",
"git-api.anchor.internal": "http://10.10.0.7:3001", "blog.example": "http://127.0.0.1:2368"}
if got := served(); !reflect.DeepEqual(got, want) {
t.Fatalf("serving %v, want %v", got, want)
}
if !held.eligibleForACME("git.example") || held.eligibleForACME("git-api.anchor.internal") {
t.Error("a public name is not certified publicly, or an internal-only one is")
}
// Each refusal said with why, and each name served said once.
sort.Strings(said)
for _, w := range []string{
"route.refused anchor gitea refused not a route this contract reads: json: unknown field \"host\"",
"route.refused anchor lost it runs on laptop and its plan says no address on the private network for it",
"route.refused anchor older its plan names this proxy for its route and carries nothing to serve",
"route.served anchor blog.example blog ",
"route.served anchor git-api.anchor.internal gitea ",
} {
if !slices.Contains(said, w) {
t.Errorf("not said %q, said:\n%s", w, strings.Join(said, "\n"))
}
}
if len(said) != 7 {
t.Errorf("said %d events, want 3 refusals and 4 names served:\n%s", len(said), strings.Join(said, "\n"))
}
for _, row := range []struct {
name, key, value string
served []string
withdrawn []string
say []string
}{
{"a route arrives", "anchor.new", plan("anchor", "new", route+contributed("", "", `{"name":"new.example","port":81}`)+`}`),
[]string{"new.example"}, nil, []string{"route.served anchor new.example new "}},
{"an unreadable plan keeps what it gave", "laptop.gitea", `{"v":1,`, []string{"git.example"}, nil, nil},
{"a deleted plan withdraws its routes", "laptop.gitea", "", nil, []string{"git.example", "git-api.anchor.internal"},
[]string{"route.withdrawn anchor git-api.anchor.internal ", "route.withdrawn anchor git.anchor.internal ",
"route.withdrawn anchor git.example "}},
} {
said = nil
change := stdio.StateChange{Key: row.key, Op: "put", Value: json.RawMessage(row.value)}
if row.value == "" {
change = stdio.StateChange{Key: row.key, Op: "delete"}
}
if err := plans.handle(change); err != nil {
t.Fatal(err)
}
got := served()
for _, host := range row.served {
if _, ok := got[host]; !ok {
t.Errorf("%s: %s is not served (%v)", row.name, host, got)
}
}
for _, host := range row.withdrawn {
if _, ok := got[host]; ok {
t.Errorf("%s: %s is still served", row.name, host)
}
}
if !slices.Equal(said, row.say) {
t.Errorf("%s: said %q, want %q", row.name, said, row.say)
}
}
held = behind(t)
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code != http.StatusNotFound {
t.Errorf("before the placement is read, a machine of the mesh was served an internal-only name: %d", code)
}
laptop, err := delivery.PlacementValues(delivery.Placement{Node: "laptop", Decided: delivery.NodeDecided{Address: "10.10.0.7"}})
if err != nil {
t.Fatal(err)
}
placement := &fakePlans{current: map[string]string{}}
for k, v := range laptop {
placement.current[k] = string(v)
}
if err := followInside(placement, held); err != nil {
t.Fatal(err)
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("a machine the placement decides an address for was refused the internal-only route")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.8:1"); code != http.StatusNotFound {
t.Error("a machine the placement does not name was served the internal-only route")
}
if err := placement.handle(stdio.StateChange{Key: "laptop.node", Op: "put", Value: json.RawMessage(`{`)}); err != nil {
t.Fatal(err)
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("an unreadable placement key changed who is inside")
}
}
modules/route-proxy/cmd/route-proxy/routes_test.go
-589
View File
@@ -1,589 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
)
// routesFrom is the table a test's routes make, written as `{"given":[{"from","node","at","values"}…]}`: the
// contributions the plans give the proxy, each read with route-proxy's contract as routesOfPlan reads it, and one the
// contract refuses left out, as routesOfPlan leaves it out.
func routesFrom(body string) (map[string][]rule, map[string]bool, error) {
var said struct {
Given []struct {
From, Node, At string
Values json.RawMessage
} `json:"given"`
}
if err := json.Unmarshal([]byte(body), &said); err != nil {
return nil, nil, err
}
var given []contribution
for _, g := range said.Given {
route, err := contract.DecodeRequest(g.Values)
if err != nil {
continue
}
given = append(given, contribution{From: g.From, Node: g.Node, At: g.At, Route: route})
}
routes, public := routesOf(given)
return routes, public, nil
}
// plain is the table an ordinary set of routes makes: one host, one target, no policy.
func plain(routes map[string]string) map[string][]rule {
out := map[string][]rule{}
for host, target := range routes {
out[host] = []rule{{target: target}}
}
return out
}
// allPublic is every host in a routes map, ACME-eligible — the ordinary case for a test with no
// internal-name alias of its own to distinguish.
func allPublic(routes map[string][]rule) map[string]bool {
out := map[string]bool{}
for host := range routes {
out[host] = true
}
return out
}
// targetOf is where a host's first matching rule sends a request.
func targetOf(routes map[string][]rule, host string) string {
if rules := routes[host]; len(rules) > 0 {
return rules[0].target
}
return ""
}
// A route is a grant: the consumer supplies a target, and where that machine is comes from the
// mesh rather than from a naming convention the proxy has to know.
func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
routes, _, err := routesFrom(`{"contributions":1,"requirement":"route","given":[
{"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}}
]}`)
if err != nil {
t.Fatal(err)
}
// Lower-cased, because a Host header is not case-sensitive and a route that only answers the
// spelling in the manifest answers half the requests made to it.
if targetOf(routes, "app.example") != "http://laptop.internal:8080" {
t.Fatalf("the route does not point at the consumer: %v", routes)
}
}
// A route with an internal-name alias is reachable under both hostnames, pointed at the same
// target — the same convenience a predecessor proxy gave for reaching a service over the VPN
// without a public TLS round trip.
func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`)
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.example") != "http://anchor.internal:8080" {
t.Fatalf("the public name does not point at the consumer: %v", routes)
}
if targetOf(routes, "app.anchor.internal") != "http://anchor.internal:8080" {
t.Fatalf("the internal alias does not point at the same consumer: %v", routes)
}
if !public["app.example"] {
t.Errorf("the public name is not eligible for a certificate: %v", public)
}
if public["app.anchor.internal"] {
t.Errorf("the internal alias is eligible for a certificate no public CA could ever issue: %v",
public)
}
}
// A route whose endpoint reaches only the private network carries an internal name and no public
// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing —
// skipping it left every internal-only module unreachable by name (novox/hq issue 191).
func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}}
]}`)
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.anchor.internal") != "https://anchor.internal:8443" {
t.Fatalf("the internal-only route is not served: %v", routes)
}
if len(routes) != 1 {
t.Errorf("an internal-only route made hosts it never named: %v", routes)
}
if len(public) != 0 {
t.Errorf("an internal-only route made a name eligible for a public certificate: %v", public)
}
held := newTable()
held.set(routes, public)
if err := onlyInternalNamesTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err != nil {
t.Errorf("the internal authority refused the internal-only route's name: %v", err)
}
if err := onlyWhatTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err == nil {
t.Error("a public certificate was ordered for an internal-only name")
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
{"from":"app","node":"anchor","values":{"name":"app.example","port":8080}}
]}`)
if err != nil {
t.Fatal(err)
}
if len(routes) != 1 {
t.Fatalf("a route with no internal-name grew a second host: %v", routes)
}
}
// A route is dialled where the machine publishes its module's endpoint, as the module's declaration publishes it (issue
// 774, research 072 row 34; 2026-10-31: gitea's 3000, published at 20000, and grafana's 3000 on another machine,
// published at 20012, answered 502): over loopback beside the proxy, which is the only thing that works without a
// private network, and at its machine's address otherwise. Each plan is composed by the SDK as the seat
// mesh-provisioner composes it, its ports as compose's portsFor gives them (the placement's for a port the mesh may
// move, the manifest's own otherwise), and read as this proxy reads it: the live shapes the old proxy served alike.
func TestAComposedPlanIsDialledWhereTheMachinePublishesIt(t *testing.T) {
proxy := catalogue.Manifest{Module: "route-proxy", Version: "1",
Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh, AnsweredBy: catalogue.AnsweredByHelper}}}
world := catalogue.World{Offered: map[string][]catalogue.Provider{
"route": {{Node: "anchor", At: "anchor.internal", Module: "route-proxy"}}}}
for _, c := range []struct {
name, node, module string
port int // the endpoint's port, as listens says it
published []any // the container's ports; nil for the host's network
placed map[int]int // what mesh-placement holds for the module
target string
}{
{"beside the proxy, moved by the placement (gitea)", "anchor", "gitea", 3000, []any{"3000"}, map[int]int{3000: 20000},
"http://127.0.0.1:20000"},
{"on another machine, moved by the placement (grafana)", "home", "grafana", 3000, []any{"3000"},
map[int]int{3000: 20012}, "http://home.internal:20012"},
{"a bare port the placement holds none for (influxdb)", "home", "influxdb", 8086, []any{"8086"}, nil,
"http://home.internal:8086"},
{"a fixed mapping (mailu 7080:80)", "anchor", "mailu", 7080, []any{"25", "7080:80"}, nil, "http://127.0.0.1:7080"},
{"a fixed mapping the placement names another port of (photos 9102:9000)", "anchor", "photos", 9102,
[]any{"9102:9000"}, map[int]int{9000: 20011}, "http://127.0.0.1:9102"},
{"the host's network (home-assistant)", "anchor", "home-assistant", 8123, nil, nil, "http://127.0.0.1:8123"},
} {
container := map[string]any{"type": "container", "name": c.module, "image": "x"}
if c.published != nil {
container["ports"] = c.published
} else {
container["network"] = "host"
}
module := catalogue.Manifest{Module: c.module, Version: "1", Requires: []string{"route"},
Listens: []catalogue.Listening{{Name: "web", Port: c.port, Protocol: "tcp", From: "mesh"}},
Contributes: map[string]map[string]any{"route": {"label": c.module, "endpoint": "web"}},
Resources: []map[string]any{container}}
node := catalogue.Node{Name: c.node, At: c.node + ".internal", PublicDomain: "example.org",
Capabilities: map[string]bool{"container-runtime": true}}
r, err := catalogue.Resolve(map[string]catalogue.Manifest{module.Module: module, proxy.Module: proxy},
[]string{module.Module}, node, world)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
r.Account = "op"
for i, n := range r.Needs {
r.Needs[i].AnsweredByHelper = n.Name == "route" // as the plan sets it (compose's answeredByHelper)
}
ports := map[int]int{}
for _, l := range module.Listens { // compose's portsFor
where, mayAssign := module.MachineSide(l.Port)
if at, known := c.placed[l.Port]; mayAssign && known {
where = at
}
ports[l.Port] = where
}
composed, err := r.Compose(catalogue.Rendering{Hands: true, Provisioner: "mesh-provisioner-client",
ArtifactStore: "store:5000", Planned: true, Ports: map[string]map[int]int{c.module: ports}})
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
value, err := provisions.EncodePlan(provisions.Plan{Node: c.node, Module: c.module, Plan: composed.Plans[c.module]})
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
given, refused, err := routesOfPlan("anchor", c.node+"."+c.module, value)
if err != nil || len(refused) != 0 {
t.Fatalf("%s: the plan gives %v, refused %v (%v)", c.name, given, refused, err)
}
routes, _ := routesOf(given)
if got := targetOf(routes, c.module+".example.org"); got != c.target {
t.Errorf("%s: dialled at %q, want %q", c.name, got, c.target)
}
}
}
// A route may name a target reached over https, for a backend that terminates its own TLS — the
// shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise.
func TestARouteMayTargetHttps(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
{"from":"mail","node":"anchor","at":"anchor.internal",
"values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}}
]}`)
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "mail.example") != "https://anchor.internal:7443" {
t.Fatalf("an https target was not built as one: %v", routes)
}
if !routes["mail.example"][0].insecure {
t.Fatal("insecure was declared and not carried onto the rule")
}
}
// End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still
// reached when the route declared `insecure`, and the response comes back through unmodified.
func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) {
workload := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("the workload, over its own TLS"))
}))
defer workload.Close()
target := strings.TrimPrefix(workload.URL, "https://")
held := newTable()
routes := map[string][]rule{"mail.example": {{target: "https://" + target, insecure: true}}}
held.set(routes, allPublic(routes))
proxy := httptest.NewServer(handler(held))
defer proxy.Close()
asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil)
if err != nil {
t.Fatal(err)
}
asked.Host = "mail.example"
answer, err := http.DefaultClient.Do(asked)
if err != nil {
t.Fatal(err)
}
defer answer.Body.Close()
if answer.StatusCode != http.StatusOK {
t.Fatalf("an insecure https backend was not reached: %d", answer.StatusCode)
}
}
// End to end through the proxy itself: a request for the name reaches the workload, and a name
// nobody asked for is refused in a way that says what IS served.
func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("the workload"))
}))
defer workload.Close()
target := strings.TrimPrefix(workload.URL, "http://")
host, port, _ := strings.Cut(target, ":")
held := newTable()
held.set(plain(map[string]string{"app.example": "http://" + host + ":" + port}), allPublic(plain(map[string]string{"app.example": "http://" + host + ":" + port})))
proxy := httptest.NewServer(handler(held))
defer proxy.Close()
asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil)
if err != nil {
t.Fatal(err)
}
asked.Host = "app.example"
answer, err := http.DefaultClient.Do(asked)
if err != nil {
t.Fatal(err)
}
defer answer.Body.Close()
if answer.StatusCode != http.StatusOK {
t.Fatalf("a request for a served name got %d", answer.StatusCode)
}
// And a name that is not served says which are — a route withdrawn and a name that never
// existed are different things, and a bare 404 makes an operator go and read the mesh.
other, _ := http.NewRequest(http.MethodGet, proxy.URL, nil)
other.Host = "nobody.example"
refused, err := http.DefaultClient.Do(other)
if err != nil {
t.Fatal(err)
}
defer refused.Body.Close()
if refused.StatusCode != http.StatusNotFound {
t.Fatalf("a name nobody asked for got %d", refused.StatusCode)
}
body := make([]byte, 256)
n, _ := refused.Body.Read(body)
if !strings.Contains(string(body[:n]), "app.example") {
t.Fatalf("the refusal does not say what is served: %s", body[:n])
}
}
// The file is the whole truth about who has a route, so the table replaces rather than merges.
//
// Merging would keep serving a name whose module was unassigned — the stale-route fault
// 08-connectivity lists as open, reintroduced one level down. A stale public name pointing at
// nothing fails more visibly than a stale grant, which is exactly why it must not survive.
func TestWithdrawingARouteStopsServingIt(t *testing.T) {
held := newTable()
initial := plain(map[string]string{
"going.example": "http://a.internal:80",
"staying.example": "http://b.internal:80",
})
held.set(initial, allPublic(initial))
held.set(plain(map[string]string{"staying.example": "http://b.internal:80"}), allPublic(plain(map[string]string{"staying.example": "http://b.internal:80"})))
if _, still := held.find("going.example", "/"); still {
t.Fatal("a route whose module was unassigned is still served")
}
if _, kept := held.find("staying.example", "/"); !kept {
t.Fatal("withdrawing one route took another with it")
}
}
// A Host header carries a port and the name does not.
func TestARequestNamingAPortStillFindsItsRoute(t *testing.T) {
held := newTable()
held.set(plain(map[string]string{"app.example": "http://a.internal:8080"}), allPublic(plain(map[string]string{"app.example": "http://a.internal:8080"})))
if _, found := held.find("app.example:8080", "/"); !found {
t.Fatal("a request to app.example:8080 did not find the route for app.example")
}
}
// Defends novox/hq 04-ISSUES/004: issuance targets staging unless something says otherwise.
//
// The failure this guards is not a broken proxy. It is a working one that quietly spends a
// production quota which does not replenish for a week, on exactly the work most likely to
// iterate.
func TestTheIssuerIsStagingUnlessNamed(t *testing.T) {
t.Setenv("ACME_DIRECTORY", "")
if got := issuer(); !strings.Contains(got, "staging") {
t.Fatalf("with nothing set the issuer is %q, and a default that spends production quota "+
"is a default nobody chose", got)
}
t.Setenv("ACME_DIRECTORY", "https://acme-v02.api.letsencrypt.org/directory")
if got := issuer(); strings.Contains(got, "staging") {
t.Fatalf("an issuer was named explicitly and %q was used instead", got)
}
}
// A certificate is only ever asked for on a name the mesh routes here.
//
// **Without this, anything that can reach the port spends the quota.** A scan sending arbitrary
// names, or one misconfigured client, becomes a stream of failed orders against the account's
// rate limit — and the proxy would look healthy throughout.
func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) {
held := newTable()
held.set(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"}), allPublic(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"})))
policy := onlyWhatTheMeshSaid(held)
if err := policy(context.Background(), "photos.example"); err != nil {
t.Errorf("a name the mesh routes here was refused a certificate: %v", err)
}
for _, name := range []string{"unknown.example", "", "photos.example.evil"} {
if err := policy(context.Background(), name); err == nil {
t.Errorf("a certificate would be ordered for %q, which the mesh never mentioned", name)
}
}
}
// A certificate is asked for on a route's public name, never on its internal-network alias — no
// public CA can validate a private name, and asking anyway would only spend the account's rate
// limit on an order that can never succeed.
func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`)
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, public)
policy := onlyWhatTheMeshSaid(held)
if err := policy(context.Background(), "app.example"); err != nil {
t.Errorf("the route's public name was refused a certificate: %v", err)
}
if err := policy(context.Background(), "app.anchor.internal"); err == nil {
t.Error("a certificate was ordered for the internal alias, which no public CA can validate")
}
}
// A certificate is asked of the *internal* authority only for a name that is routed here and is
// not a route's own public name — the internal-network alias, never the route it accompanies.
func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`)
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, public)
policy := onlyInternalNamesTheMeshSaid(held)
if err := policy(context.Background(), "app.anchor.internal"); err != nil {
t.Errorf("the internal alias was refused by its own authority: %v", err)
}
if err := policy(context.Background(), "app.example"); err == nil {
t.Error("the internal authority certified a route's public name, which the public authority already covers")
}
if err := policy(context.Background(), "unrouted.internal"); err == nil {
t.Error("the internal authority certified a name nobody routed here")
}
}
// A route withdrawn stops being certifiable, without the proxy restarting.
func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) {
held := newTable()
held.set(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"}), allPublic(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"})))
policy := onlyWhatTheMeshSaid(held)
if err := policy(context.Background(), "photos.example"); err != nil {
t.Fatal(err)
}
held.set(nil, nil)
if err := policy(context.Background(), "photos.example"); err == nil {
t.Fatal("a withdrawn route can still order certificates, so the policy read a copy taken " +
"once rather than what is served now")
}
}
// A route may say the largest body it carries, and the proxy holds requests to it.
//
// The registry is why: image layers arrive as single requests of gigabytes, and a proxy's own
// default refuses them long before the workload is reached. It is configuration beside `insecure`,
// not a fifth policy — novox/hq ADR 0108 closed that set at four.
func TestARouteMayLimitTheBodyItCarries(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
{"from":"registry","node":"anchor","at":"anchor.internal",
"values":{"name":"images.example","port":5000,"max-request-body":21474836480}}
]}`)
if err != nil {
t.Fatal(err)
}
rules := routes["images.example"]
if len(rules) != 1 {
t.Fatalf("the route is not served once: %v", routes)
}
if rules[0].maxRequestBody != 21474836480 {
t.Fatalf("the limit did not survive the contribution: %d", rules[0].maxRequestBody)
}
}
// Saying nothing leaves the route unlimited, which is what every route already got.
func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"name":"app.example","port":8080}}
]}`)
if err != nil {
t.Fatal(err)
}
if got := routes["app.example"][0].maxRequestBody; got != 0 {
t.Fatalf("a route that asked for no limit got one: %d", got)
}
}
// A request larger than the route carries is refused by the proxy, with the limit named, and the
// workload never sees it. A request within it is proxied normally.
func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
var reached int
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
reached++
fmt.Fprintf(w, "carried %d bytes", len(body))
}))
defer workload.Close()
at := strings.TrimPrefix(workload.URL, "http://")
host, port, _ := strings.Cut(at, ":")
routes, _, err := routesFrom(`{"given":[
{"from":"registry","node":"anchor","at":"` + host + `",
"values":{"name":"images.example","port":` + port + `,"max-request-body":8}}
]}`)
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, map[string]bool{})
proxy := httptest.NewServer(handler(held))
defer proxy.Close()
over, err := post(proxy.URL, "images.example", "123456789")
if err != nil {
t.Fatal(err)
}
defer over.Body.Close()
if over.StatusCode != http.StatusRequestEntityTooLarge {
t.Fatalf("a body over the limit answered %d, not 413", over.StatusCode)
}
if reached != 0 {
t.Fatalf("the workload was reached by a request the route said it would not carry")
}
under, err := post(proxy.URL, "images.example", "1234")
if err != nil {
t.Fatal(err)
}
defer under.Body.Close()
if under.StatusCode != http.StatusOK {
t.Fatalf("a body within the limit answered %d, not 200", under.StatusCode)
}
if reached != 1 {
t.Fatalf("the workload was not reached by a request within the limit")
}
}
// post sends a body to the proxy as the named route, since a route is found by the Host header.
func post(url, host, body string) (*http.Response, error) {
asked, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
if err != nil {
return nil, err
}
asked.Host = host
asked.Header.Set("Content-Type", "application/octet-stream")
return http.DefaultClient.Do(asked)
}
// A contribution that cannot be served is skipped rather than served wrongly: one that names nothing to be served
// under; one with no port, which would proxy to :0; a scheme that is neither http nor https, refused rather than
// guessed at; and a body limit that is not a whole positive number of bytes, since serving it without the limit would
// carry exactly what the module said not to carry, and report success doing it.
func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) {
for _, row := range []struct{ why, given, served string }{
{"neither name", `{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}`, ""},
{"no port", `{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}`, "fine.example"},
{"an unknown scheme", `{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}`, ""},
{"a limit in words", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":"lots"}}`, ""},
{"a negative limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":-1}}`, ""},
{"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""},
{"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""},
} {
routes, public, err := routesFrom(`{"given":[` + row.given + `]}`)
if err != nil {
t.Fatal(err)
}
switch {
case row.served == "" && (len(routes) != 0 || len(public) != 0):
t.Errorf("%s: an unusable contribution was served: %v %v", row.why, routes, public)
case row.served != "" && (len(routes) != 1 || targetOf(routes, row.served) == ""):
t.Errorf("%s: an unusable contribution was served beside the usable one: %v", row.why, routes)
}
}
}
modules/route-proxy/contract/contract.go
-77
View File
@@ -1,77 +0,0 @@
// Package contract is route-proxy's contract for the provision `route`: what a consumer asks a route with, what the
// proxy keeps on the bus about what it serves, the events it says, and its tools' arguments (issue 774; research 072
// row 34; the operator, 2026-10-31: "the route-proxy should define its own contract for the seat (with the message
// contracts of course) which other modules can use").
//
// **Published by the repository that holds route-proxy, as its own Go module**, versioned by tags of the form
// `modules/route-proxy/contract/vX.Y.Z`, the subdirectory form Go's module system expects, the same way mesh-sdk is
// published (`go/vX.Y.Z`). A module that asks a route, reads what the proxy serves or hears its events imports this
// package; nothing imports the repository. Standard library only, so a consumer takes nothing else with it.
//
// **How a route travels.** A module that must be reachable requires `route` and says, in its manifest's
// `contributes.route`, one Request, or several by local name; Schema is that object's JSON schema. The controller
// composes each (its label joined to the node's domains into `name` and `internal-name`, its endpoint's port into
// `port`) and the seat mesh-provisioner carries the result in the consumer's plan entry as an opaque payload
// (mesh-sdk catalogue.PlanProvision.Contributed, in the generic catalogue.ContributedForm). Neither the seat nor its
// client knows a field of a route: route-proxy decodes each contribution with DecodeRequest, and refuses, saying why
// (EventRefused), what this contract does not admit.
//
// **Two name spaces, two authorities** (novox/hq ADR 0056, ADR 0138). `name` is public: served to anyone and
// certified by the public ACME directory (Let's Encrypt). `internal-name` is served only inside the private network
// and certified by the mesh's own authority. A route carries either, or both.
package contract
// Module is the module that provides a route, and Provision what it provides: what a plan's entry names when
// route-proxy is a route's provider.
const (
Module = "route-proxy"
Provision = "route"
)
// Version is the version of the Request form this package reads and writes.
const Version = 1
// The events route-proxy says (its manifest's `emits`), on mesh.mod.route-proxy.event.<type>, each with an Event.
const (
// EventServed: a name is now served by this proxy (it was not, or it is now served from other rules).
EventServed = "route.served"
// EventWithdrawn: a name this proxy served is served no more, because no plan gives it any longer.
EventWithdrawn = "route.withdrawn"
// EventRefused: a plan gives this proxy a route it does not serve, and Why says why.
EventRefused = "route.refused"
)
// Event is the body of every event route-proxy says: the proxy's machine, the name (empty for a refusal that named
// none), how far it reaches, and who asked it: the consumer module, its machine and the local name of its
// contribution. Why is set on a refusal alone.
type Event struct {
Proxy string `json:"proxy"`
Name string `json:"name,omitempty"`
Reach string `json:"reach,omitempty"`
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
Local string `json:"local,omitempty"`
Why string `json:"why,omitempty"`
}
// The tools route-proxy serves (its manifest's `tools`), each answering from its states RoutesState and UnroutedState.
const (
// ToolRoutes lists every name served, with RoutesArgs.
ToolRoutes = "route_proxy_routes"
// ToolRoute is one name in full, with RouteArgs.
ToolRoute = "route_proxy_route"
// ToolUnrouted lists the names asked for and not served, with RoutesArgs.
ToolUnrouted = "route_proxy_unrouted"
)
// RoutesArgs are ToolRoutes' and ToolUnrouted's arguments: Node narrows to the proxy on one machine, every machine
// when empty.
type RoutesArgs struct {
Node string `json:"node,omitempty"`
}
// RouteArgs are ToolRoute's arguments: the name, such as app.example.org, and optionally the machine whose proxy.
type RouteArgs struct {
Name string `json:"name"`
Node string `json:"node,omitempty"`
}
modules/route-proxy/contract/contract_test.go
-85
View File
@@ -1,85 +0,0 @@
package contract
import (
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
)
func limit(n int64) *int64 { return &n }
// A route is read strictly and whole (issue 774): what the contract names is admitted, what it does not, or a route
// it cannot serve, is refused naming why; and the published schema names exactly the fields the decoder reads.
func TestARouteIsReadStrictly(t *testing.T) {
for _, c := range []struct {
raw, refused string
want Request
}{
{raw: `{"name":"app.example.org","internal-name":"app.anchor.internal","label":"app","endpoint":"web","port":8080}`,
want: Request{Name: "app.example.org", InternalName: "app.anchor.internal", Label: "app", Endpoint: "web", Port: 8080}},
{raw: `{"internal-name":"app.anchor.internal","port":443,"scheme":"https","insecure":true,"max-request-body":1048576}`,
want: Request{InternalName: "app.anchor.internal", Port: 443, Scheme: "https", Insecure: true, MaxRequestBody: limit(1048576)}},
{raw: `{"name":"git.example.org","path":"/api/internal","deny":true,"priority":100000}`,
want: Request{Name: "git.example.org", Path: "/api/internal", Deny: true, Priority: 100000}},
{raw: `{"name":"old.example.org","redirect":"https://new.example.org"}`,
want: Request{Name: "old.example.org", Redirect: "https://new.example.org"}},
{raw: `{"name":"admin.example.org","port":80,"auth":"/run/secrets/admin"}`,
want: Request{Name: "admin.example.org", Port: 80, Auth: "/run/secrets/admin"}},
{raw: `{"label":"app","port":8080}`, refused: "names nothing"},
{raw: `{"name":"app.example.org"}`, refused: "gives no port"},
{raw: `{"name":"app.example.org","port":70000}`, refused: "not a port"},
{raw: `{"name":"app.example.org","port":80.5}`, refused: "not a route"},
{raw: `{"name":"app.example.org","port":"80"}`, refused: "not a route"},
{raw: `{"name":"app.example.org","port":80,"scheme":"ftp"}`, refused: "neither http nor https"},
{raw: `{"name":"app.example.org","port":80,"max-request-body":-1}`, refused: "max-request-body"},
{raw: `{"name":"app.example.org","port":80,"max-request-body":0}`, refused: "max-request-body"},
{raw: `{"name":"app.example.org","port":80,"auth":"admin:$2y$05$abc"}`, refused: "credential"},
{raw: `{"name":"app.example.org","port":80,"host":"x"}`, refused: "unknown field \"host\""},
{raw: `{"name":"app.example.org","port":80} {}`, refused: "more than one"},
} {
got, err := DecodeRequest([]byte(c.raw))
if c.refused != "" {
if err == nil || !strings.Contains(err.Error(), c.refused) {
t.Errorf("DecodeRequest(%s) = %+v, %v; want refused for %q", c.raw, got, err, c.refused)
}
continue
}
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("DecodeRequest(%s) = %+v, %v; want %+v", c.raw, got, err, c.want)
continue
}
// What it encodes, it reads back the same.
if raw, err := got.Encode(); err != nil {
t.Errorf("Encode(%+v): %v", got, err)
} else if again, err := DecodeRequest(raw); err != nil || !reflect.DeepEqual(again, got) {
t.Errorf("Encode then DecodeRequest of %+v = %+v, %v", got, again, err)
}
}
if (Request{Path: "api"}).Prefix() != "/api" || (Request{}).Prefix() != "" || (Request{Scheme: " HTTPS"}).SchemeOrDefault() != "https" ||
(Request{}).SchemeOrDefault() != "http" || (Request{Redirect: "x"}).Forwards() {
t.Error("Prefix, SchemeOrDefault or Forwards says otherwise than the contract")
}
var schema struct {
Properties map[string]json.RawMessage `json:"properties"`
AdditionalProperties bool `json:"additionalProperties"`
}
if err := json.Unmarshal(Schema, &schema); err != nil || schema.AdditionalProperties {
t.Fatalf("the schema is not a closed object: %v", err)
}
var tags []string
for f := range reflect.TypeFor[Request]().Fields() {
tags = append(tags, strings.Split(f.Tag.Get("json"), ",")[0])
}
var said []string
for name := range schema.Properties {
said = append(said, name)
}
slices.Sort(tags)
slices.Sort(said)
if !slices.Equal(tags, said) {
t.Fatalf("the schema names %v, the decoder reads %v", said, tags)
}
}
modules/route-proxy/contract/go.mod
-3
View File
@@ -1,3 +0,0 @@
module git.novox.be/novox/mesh-controller/modules/route-proxy/contract
go 1.26.0
modules/route-proxy/contract/request.go
-128
View File
@@ -1,128 +0,0 @@
package contract
import (
"bytes"
_ "embed"
"encoding/json"
"errors"
"fmt"
"strings"
)
// Request is one route a consumer asks for, as its manifest's `contributes.route` says it and the controller composes
// it: the object Schema describes. Every field is optional to the decoder; Validate says which a route needs.
type Request struct {
// Name is the public name, served to anyone and certified by the public ACME directory; InternalName the name on
// the private network, served inside it alone and certified by the mesh's own authority. The controller composes
// both from Label and the node's domains, as far as the endpoint reaches (novox/hq ADR 0138); a manifest writes a
// label, never a domain.
Name string `json:"name,omitempty"`
InternalName string `json:"internal-name,omitempty"`
Label string `json:"label,omitempty"`
// Endpoint names the manifest's endpoint the route serves; the controller fills Port from it.
Endpoint string `json:"endpoint,omitempty"`
Port int `json:"port,omitempty"`
// Path is the path prefix the rule is scoped to, every path when empty; Priority which rule wins where two match,
// higher first.
Path string `json:"path,omitempty"`
Priority int `json:"priority,omitempty"`
// Scheme is how the consumer is reached, http (the default) or https; Insecure skips verifying its certificate,
// for a backend that terminates TLS with its own.
Scheme string `json:"scheme,omitempty"`
Insecure bool `json:"insecure,omitempty"`
// MaxRequestBody is the largest body, in bytes, the route carries; no limit when absent. Said, it is whole and
// positive: a limit the proxy cannot keep is a route it does not serve.
MaxRequestBody *int64 `json:"max-request-body,omitempty"`
// The policy of the rule, a closed set (novox/hq ADR 0108): Deny refuses every request, Redirect answers with a
// permanent redirect to that address, and Auth names the file of the secret a request must present, never the
// secret itself.
Deny bool `json:"deny,omitempty"`
Redirect string `json:"redirect,omitempty"`
Auth string `json:"auth,omitempty"`
}
// Schema is Request's JSON schema, published with this package for what checks a manifest's `contributes.route`.
//
//go:embed route.schema.json
var Schema []byte
// DecodeRequest reads one contribution's object strictly: no field this contract does not name, one document, and
// a route Validate admits. What it refuses, it refuses whole, naming why: a route is never served half.
func DecodeRequest(raw []byte) (Request, error) {
var r Request
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&r); err != nil {
return Request{}, fmt.Errorf("not a route this contract reads: %w", err)
}
if dec.More() {
return Request{}, errors.New("not a route this contract reads: more than one document")
}
if err := r.Validate(); err != nil {
return Request{}, err
}
return r, nil
}
// Encode is r as a contribution's object, refused where DecodeRequest would refuse it.
func (r Request) Encode() ([]byte, error) {
if err := r.Validate(); err != nil {
return nil, err
}
return json.Marshal(r)
}
// Validate says what makes r no route: no name of either kind; a port outside 1-65535, or none where the rule
// forwards (neither Deny nor Redirect); a scheme other than http or https; a body limit that is not positive; and a
// credential written where a secret's file is to be named (novox/hq ADR 0108: refused rather than tolerated).
func (r Request) Validate() error {
if strings.TrimSpace(r.Name) == "" && strings.TrimSpace(r.InternalName) == "" {
return errors.New("the route names nothing to be served under: neither name nor internal-name")
}
if r.Port < 0 || r.Port > 65535 {
return fmt.Errorf("the route's port %d is not a port", r.Port)
}
if r.Forwards() && r.Port == 0 {
return errors.New("the route forwards and gives no port")
}
switch strings.ToLower(strings.TrimSpace(r.Scheme)) {
case "", "http", "https":
default:
return fmt.Errorf("the route's scheme %q is neither http nor https", r.Scheme)
}
if r.MaxRequestBody != nil && *r.MaxRequestBody <= 0 {
return fmt.Errorf("the route's max-request-body %d is not a positive number of bytes", *r.MaxRequestBody)
}
if LooksLikeACredential(r.Auth) {
return errors.New("the route's auth holds a credential rather than the name of a secret's file (novox/hq ADR 0108)")
}
return nil
}
// Forwards is whether the rule sends a request on to the consumer, rather than answering it itself.
func (r Request) Forwards() bool { return !r.Deny && r.Redirect == "" }
// Prefix is the path prefix the rule is scoped to, beginning with a slash, or "" for every path.
func (r Request) Prefix() string {
p := strings.TrimSpace(r.Path)
if p != "" && !strings.HasPrefix(p, "/") {
p = "/" + p
}
return p
}
// SchemeOrDefault is how the consumer is reached: the scheme, lower-cased, http when none is said.
func (r Request) SchemeOrDefault() string {
if s := strings.ToLower(strings.TrimSpace(r.Scheme)); s != "" {
return s
}
return "http"
}
// LooksLikeACredential is the check that keeps a secret out of a declaration: a value holding a `:` (the htpasswd
// separator) or opening with a bcrypt identifier is a credential, not a path, and no path the mesh writes needs
// either. It errs towards refusing.
func LooksLikeACredential(v string) bool {
v = strings.TrimSpace(v)
return strings.Contains(v, ":") || strings.HasPrefix(v, "$2")
}
modules/route-proxy/contract/route.schema.json
-28
View File
@@ -1,28 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://git.novox.be/novox/mesh-controller/modules/route-proxy/contract/route.schema.json",
"title": "route",
"description": "One route a consumer asks route-proxy for, as a manifest's contributes.route says it (one, or several by local name) and the controller composes it. Version 1 of route-proxy's contract.",
"type": "object",
"additionalProperties": false,
"anyOf": [
{"required": ["name"]},
{"required": ["internal-name"]},
{"required": ["label"]}
],
"properties": {
"name": {"type": "string", "description": "the public name, served to anyone and certified by the public ACME directory; composed from label"},
"internal-name": {"type": "string", "description": "the name on the private network, served inside it alone and certified by the mesh's own authority; composed from label"},
"label": {"type": "string", "description": "the subdomain the module chooses; joined to the node's domains by the controller"},
"endpoint": {"type": "string", "description": "the manifest's endpoint the route serves; the controller fills port from it"},
"port": {"type": "integer", "minimum": 1, "maximum": 65535},
"path": {"type": "string", "description": "the path prefix the rule is scoped to; every path when absent"},
"priority": {"type": "integer", "description": "which rule wins where two match, higher first"},
"scheme": {"enum": ["http", "https"], "description": "how the consumer is reached; http when absent"},
"insecure": {"type": "boolean", "description": "skip verifying the consumer's certificate over https"},
"max-request-body": {"type": "integer", "minimum": 1, "description": "the largest body, in bytes, the route carries"},
"deny": {"type": "boolean", "description": "refuse every request matching the rule"},
"redirect": {"type": "string", "description": "answer with a permanent redirect to this address"},
"auth": {"type": "string", "description": "the file of the secret a request must present, never the secret itself"}
}
}
modules/route-proxy/contract/state.go
-152
View File
@@ -1,152 +0,0 @@
package contract
// What the web front keeps on the bus about the names it serves, and about the names it was asked for and does not
// serve (issue 774). The proxy writes it, on change, in its states `routes` and `unrouted`, announced readable; the
// module's tools, and any module that reads them, read it from there, so the question "is this site reachable, and
// is its certificate good" is answered from the bus rather than with curl.
//
// A key is `<node>.<name>`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds
// no dot, so the first dot splits them.
import (
"net"
"regexp"
"strings"
"time"
)
// The states route-proxy keeps, by their local names (the buckets are route-proxy_<name>).
const (
// RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got.
RoutesState = "routes"
// UnroutedState holds one key per name the proxy answered "no route for" with, and when it was last asked.
UnroutedState = "unrouted"
// BoundState is the provider helper's bucket of consumer entries (mesh-sdk go/provider).
BoundState = "bound"
)
// The words a route's reach and a certificate's authority are said in.
const (
ReachPublic = "public"
ReachInternal = "internal"
)
// The words a certificate's outcome is said in.
const (
// Issued: the authority issued it while this proxy ran, at RenewedAt.
Issued = "issued"
// Cached: this proxy served it from what it kept, and saw no issuance of it since it started.
Cached = "cached"
// Failed: the last attempt to have one failed, Error says why.
Failed = "failed"
)
// Target is one rule a name is served by: what a request matching it reaches, or what it is answered with.
type Target struct {
// Path is the path prefix the rule is scoped to; empty is every path.
Path string `json:"path,omitempty"`
// Module and Node are the consumer that contributed the route and the machine it runs on.
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
// Target is where the request is forwarded, scheme://address:port; empty for a rule that answers itself.
Target string `json:"target,omitempty"`
Deny bool `json:"deny,omitempty"`
Redirect string `json:"redirect,omitempty"`
// Auth says the rule asks for credentials; Sealed says why it cannot check them, and so refuses everything.
Auth bool `json:"auth,omitempty"`
Sealed string `json:"sealed,omitempty"`
}
// Certificate is what this proxy knows of a name's certificate.
type Certificate struct {
// Authority is which issuer certifies the name: public (the ACME directory the module names) or internal (the
// mesh's own).
Authority string `json:"authority"`
Issuer string `json:"issuer,omitempty"`
NotAfter *time.Time `json:"not_after,omitempty"`
// RenewedAt is when an authority last issued it to this proxy.
RenewedAt *time.Time `json:"renewed_at,omitempty"`
// Outcome is issued, cached or failed; empty until a handshake or an issuance said anything.
Outcome string `json:"outcome,omitempty"`
Error string `json:"error,omitempty"`
At *time.Time `json:"at,omitempty"`
}
// Answer is the last answer the proxy got from upstream for a name: the status it carried back, or the error that
// stood in for one (a 502).
type Answer struct {
Status int `json:"status"`
At time.Time `json:"at"`
Error string `json:"error,omitempty"`
Path string `json:"path,omitempty"`
}
// Route is one served name, the value of a key of RoutesState.
type Route struct {
Name string `json:"name"`
// Proxy is the machine whose proxy serves it.
Proxy string `json:"proxy"`
// Reach is public (a route's own name) or internal (only its internal-name, served inside the private network).
Reach string `json:"reach"`
Targets []Target `json:"targets"`
Certificate Certificate `json:"certificate"`
LastAnswer *Answer `json:"last_answer,omitempty"`
}
// Unrouted is one name the proxy answered "no route for", the value of a key of UnroutedState.
type Unrouted struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
// AskedAt is when the name was last asked for. Who asked is not kept here: the journal keeps it, for the
// machine's intrusion prevention.
AskedAt time.Time `json:"asked_at"`
// Times is how often it was asked since the proxy last started.
Times int `json:"times"`
// InternalOnly says the name is served, but only inside the private network, and was asked from outside.
InternalOnly bool `json:"internal_only,omitempty"`
// Handshake says it was asked in a TLS handshake (the name a client sent), not in an HTTP request.
Handshake bool `json:"handshake,omitempty"`
}
// OtherName is the name of the one key per proxy that counts the refusals not kept by name: names under no domain
// the proxy serves (what a scan sends), and names past the limit of new names a minute. Its key is
// `<node>._other`; no host name begins with an underscore.
const OtherName = "_other"
// Other is the value of a proxy's OtherName key.
type Other struct {
Proxy string `json:"proxy"`
// Minute is the last whole minute counted, and LastMinute how many refusals it held.
Minute time.Time `json:"minute"`
LastMinute int `json:"last_minute"`
// SinceStart is every refusal not kept by name since the proxy started.
SinceStart int `json:"since_start"`
}
// Registrable is the domain a name is registered under, as its last two labels; "" for an address or a single
// label. Two labels is wrong for a public suffix of two (co.uk); it only decides which refused names are kept by name.
func Registrable(name string) string {
if net.ParseIP(strings.Trim(name, "[]")) != nil {
return ""
}
labels := strings.Split(name, ".")
if len(labels) < 2 {
return ""
}
return strings.Join(labels[len(labels)-2:], ".")
}
var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`)
// Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request
// is whatever a client sent; one no key could hold is no site anybody runs.
func Keyable(name string) bool { return keyable.MatchString(name) && !strings.Contains(name, "..") }
// Key is the key of name served or asked at node's proxy.
func Key(node, name string) string { return node + "." + name }
// Split is a key's node and name.
func Split(key string) (node, name string) {
node, name, _ = strings.Cut(key, ".")
return node, name
}
modules/route-proxy/files/acme.env
-3
View File
@@ -1,3 +0,0 @@
ACME_DIRECTORY=https://acme-v02.api.letsencrypt.org:443/directory
ACME_ROOTS=https://acme-v02.api.letsencrypt.org:443
ACME_ROOTS_PATH=
modules/route-proxy/files/internal-acme.env
-3
View File
@@ -1,3 +0,0 @@
INTERNAL_ACME_DIRECTORY=https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:path}
INTERNAL_ACME_ROOTS=https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:roots}
INTERNAL_ACME_ROOTS_PATH=${bound:internal-acme-ca:roots}
modules/route-proxy/go.mod
-19
View File
@@ -1,19 +0,0 @@
module route-proxy
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
)
require (
git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
)
modules/route-proxy/go.sum
-32
View File
@@ -1,32 +0,0 @@
git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0 h1:89jE1oBK+l7gFJjz6wx8mUD3KxrqSDa7pUTlQkzC1do=
git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0/go.mod h1:Yr+Xp89kpMiKH6kA06EU+d9I2CyzrK9yMX7J3pSv0hE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
modules/route-proxy/module.json
-272
View File
@@ -1,272 +0,0 @@
{
"module": "route-proxy",
"version": "1",
"slug": "rproxy",
"capabilities": [
"container-runtime"
],
"provides": [
{
"name": "route",
"scope": "mesh",
"credential": {
"form": "none"
},
"identity": false,
"answered-by": "helper"
}
],
"serves": {
"route": {}
},
"requires": [
"internal-acme-ca"
],
"binds": {
"internal-acme-ca": "${dir:state}/internal-acme-ca.json"
},
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
},
"listens": [
{
"name": "http",
"port": 80,
"protocol": "tcp",
"from": "anywhere",
"why": "public HTTP, and the ACME HTTP-01 challenge answered at the name being certified"
},
{
"name": "https",
"port": 443,
"protocol": "tcp",
"from": "anywhere",
"why": "public HTTPS for every name the mesh routes here"
}
],
"data": {
"own": [
{
"id": "acme",
"path": "${dir:acme-cache}",
"class": "rebuildable",
"why": "issued certificates, issued again"
},
{
"id": "ca",
"path": "${dir:ca-dir}",
"class": "cache",
"why": "the authority's roots, fetched again at every start"
}
]
},
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
},
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
},
{
"id": "acme-cache",
"type": "directory",
"path": "/var/lib/route-proxy/acme",
"mode": "0700"
},
{
"id": "ca-dir",
"type": "directory",
"path": "/var/lib/route-proxy/ca",
"mode": "0755"
},
{
"id": "acme-env",
"type": "file",
"path": "${dir:state}/acme.env",
"mode": "0600",
"template": "files/acme.env"
},
{
"id": "internal-acme-env",
"type": "file",
"path": "${dir:state}/internal-acme.env",
"mode": "0600",
"template": "files/internal-acme.env"
},
{
"id": "trust",
"type": "container",
"name": "route-proxy-trust",
"artifact": "trust",
"run-once": true,
"network": "host",
"env-file": [
"${dir:state}/acme.env"
],
"volumes": [
"${dir:ca-dir}:/ca"
],
"args": [
"sh",
"-c",
"if [ -z \"$ACME_ROOTS_PATH\" ]; then cp /etc/ssl/certs/ca-certificates.crt /ca/root.crt; exit 0; fi; for i in $(seq 1 60); do wget -q -T 10 --no-check-certificate -O /ca/root.crt \"$ACME_ROOTS\" && grep -q 'BEGIN CERTIFICATE' /ca/root.crt && exit 0; sleep 2; done; echo \"the authority at $ACME_ROOTS did not serve its roots within two minutes\" >&2; exit 1"
],
"restart-on": [
"acme-env"
]
},
{
"id": "internal-trust",
"type": "container",
"name": "route-proxy-internal-trust",
"artifact": "trust",
"run-once": true,
"network": "host",
"env-file": [
"${dir:state}/internal-acme.env"
],
"volumes": [
"${dir:ca-dir}:/ca"
],
"args": [
"sh",
"-c",
"if [ -z \"$INTERNAL_ACME_ROOTS_PATH\" ]; then cp /etc/ssl/certs/ca-certificates.crt /ca/internal-root.crt; exit 0; fi; for i in $(seq 1 60); do wget -q -T 10 --no-check-certificate -O /ca/internal-root.crt \"$INTERNAL_ACME_ROOTS\" && grep -q 'BEGIN CERTIFICATE' /ca/internal-root.crt && exit 0; sleep 2; done; echo \"the authority at $INTERNAL_ACME_ROOTS did not serve its roots within two minutes\" >&2; exit 1"
],
"restart-on": [
"internal-acme-env"
]
},
{
"id": "program-route-proxy",
"type": "archive",
"path": "${dir:state}/route-proxy",
"artifact": "server",
"unknown": "program"
},
{
"id": "server",
"type": "container",
"name": "route-proxy",
"artifact": "trust",
"network": "host",
"env-file": [
"${dir:state}/acme.env",
"${dir:state}/internal-acme.env"
],
"volumes": [
"${dir:acme-cache}:/acme",
"${dir:ca-dir}:/ca:ro",
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:state}/route-proxy:/opt/mesh:ro"
],
"env": {
"LISTEN": ":80",
"TLS_LISTEN": ":443",
"ACME_CACHE": "/acme",
"ACME_CA_BUNDLE": "/ca/root.crt",
"INTERNAL_ACME_CA_BUNDLE": "/ca/internal-root.crt",
"MESH_BROKER_FILE": "/run/secrets/broker"
},
"args": [
"/opt/mesh/route-proxy"
],
"restart-on": [
"trust",
"acme-env",
"internal-trust",
"internal-acme-env",
"program-route-proxy"
],
"logging": "journald",
"health": {
"kind": "tcp",
"endpoint": "http",
"interval": "10s",
"timeout": "5s",
"looks": 3,
"grace": "60s"
}
}
],
"build": {
"artifacts": [
{
"name": "server",
"kind": "bundle",
"language": "go",
"system": "alpine",
"from": "cmd/route-proxy",
"binary": "route-proxy"
},
{
"name": "trust",
"kind": "upstream",
"from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
},
{
"name": "provider",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/route-provider",
"binary": "route-provider",
"loads": [
"route-provider"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
},
"jails": [
{
"name": "route-proxy",
"failregex": "^.*TLS handshake error from <HOST>:\\d+: (?:no public route for|acme/autocert: missing server name)\n ^.*refused: no route for .*, asked from <HOST>:\\d+$",
"jail": "backend = systemd\njournalmatch = CONTAINER_NAME=route-proxy\nport = http,https\nmaxretry = 10\nfindtime = 1d\nbantime = 1d"
}
],
"state": [
"bound",
"routes",
{
"name": "unrouted",
"ttl-seconds": 604800
}
],
"tools": [
"route_proxy_routes",
"route_proxy_route",
"route_proxy_unrouted"
],
"emits": [
"route.served",
"route.withdrawn",
"route.refused"
],
"replaces": {
"route_proxy_routes": [
"curl -I"
],
"route_proxy_route": [
"openssl s_client -connect"
],
"route_proxy_unrouted": [
"docker logs route-proxy"
]
},
"reads": [
"mesh-delivery-assigner.placement",
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
]
}