diff --git a/modules/route-proxy/README.md b/modules/route-proxy/README.md index 8781c9bc..15f5e5f2 100644 --- a/modules/route-proxy/README.md +++ b/modules/route-proxy/README.md @@ -90,3 +90,17 @@ A small Go program beside the proxy (`cmd/route-provider`) runs the SDK's provid and the offer says `"answered-by": "helper"`: a handed consumer (gitea's origins file) is filled by its machine's client from this module's entry, which carries the public names the controller composed for it. The module is **not** marked moved: its own files, `acme.env` and the broker secret, are composed as before, and the proxy itself is unchanged. + +## What it says on the bus, and its tools (issue 774) + +The proxy keeps two states, announced readable (`conn.Keeps`, novox/hq ADR 0495), each keyed `.`: + +| state | one key per | holds | written when | +|---|---|---|---| +| `routes` | served name | the rules it is served by (path, module, node, target, refusal, redirect, authentication); its certificate (authority, issuer, not-after, last renewal, outcome, or why it failed); the last upstream answer (status, time, error, path) | a route arrives, changes or leaves; an authority stores a certificate; a handshake fails to find one or hands out one not yet recorded; an upstream answer differs from the last one written, or the same one is a minute old | +| `unrouted` | name answered "no route for", in a request or a TLS handshake | when it was last asked, by whom, how often since the start, whether it is an internal-only name asked from outside | first asked, then at most once a minute; deleted when the name is served; kept a week | + +Nothing is written on a timer. The provider helper's binary serves the module's tools from those states, one call each: +`route_proxy_routes` (every served name with its target, certificate expiry and last answer), `route_proxy_route` +(one name in full, or that it is not served and when it was last asked for) and `route_proxy_unrouted` (the names +refused, newest first). diff --git a/modules/route-proxy/cmd/route-provider/main.go b/modules/route-proxy/cmd/route-provider/main.go index 9ae45429..8887ed6f 100644 --- a/modules/route-proxy/cmd/route-provider/main.go +++ b/modules/route-proxy/cmd/route-provider/main.go @@ -1,6 +1,6 @@ // route-provider: route-proxy's Go half, one binary the node's runtime launches and speaks MCP to over stdio -// through the Go SDK. It serves no tools; it runs the SDK's provider helper that answers the consumers of `route` -// (issue 792). The TypeScript adapter beside it keeps writing the proxy's files. +// through the Go SDK. It runs the SDK's provider helper that answers the consumers of `route` (issue 792), and serves +// the module's tools, which read what the proxy keeps on the bus (issue 774, tools.go). // // stdout is the MCP channel, so everything this module says, it says on stderr. package main @@ -9,9 +9,12 @@ import ( "context" "fmt" "os" + "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/provider" + + "route-proxy/internal/routestate" ) func say(format string, args ...any) { @@ -30,7 +33,9 @@ func main() { } }() } - if err := stdio.Serve("", nil); err != nil { + tools := front{routes: stdio.State(routestate.RoutesState), unrouted: stdio.State(routestate.UnroutedState), + now: time.Now}.tools() + if err := stdio.Serve("", tools); err != nil { say("%v", err) os.Exit(1) } diff --git a/modules/route-proxy/cmd/route-provider/provider_test.go b/modules/route-proxy/cmd/route-provider/provider_test.go index 76617f75..01b58bfc 100644 --- a/modules/route-proxy/cmd/route-provider/provider_test.go +++ b/modules/route-proxy/cmd/route-provider/provider_test.go @@ -95,7 +95,8 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { if len(provides) != 1 || provides[0].(map[string]any)["answered-by"] != "helper" { t.Fatalf("route is not said to be answered by the helper: %v", provides) } - for _, k := range []string{"build", "provides", "state", "reads"} { + // Its states and its tools are what it says on the bus, not what it runs or writes (issue 774). + for _, k := range []string{"build", "provides", "state", "reads", "tools", "replaces"} { delete(m, k) } pinned, err := json.Marshal(m) // sorted keys, compact diff --git a/modules/route-proxy/cmd/route-provider/tools.go b/modules/route-proxy/cmd/route-provider/tools.go new file mode 100644 index 00000000..5855fd93 --- /dev/null +++ b/modules/route-proxy/cmd/route-provider/tools.go @@ -0,0 +1,252 @@ +package main + +// route-proxy's tools (issue 774): the web front's command line. Each answers from what the proxy keeps on the bus +// (route-proxy_routes and route-proxy_unrouted, written on change by the proxy itself), in one call, for every +// machine running a proxy: which sites are served and where they go, whether their certificates are good, what the +// last request through each got back, and which names were asked for and are served nowhere. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "route-proxy/internal/routestate" +) + +// The tools' names, as the manifest declares them. +const ( + RoutesTool = "route_proxy_routes" + RouteTool = "route_proxy_route" + UnroutedTool = "route_proxy_unrouted" +) + +// reader is one of the module's states, as stdio.KeptState reads it. +type reader interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// front is what the tools read: the routes and the refusals, and the clock a certificate's days left are counted by. +type front struct { + routes, unrouted reader + now func() time.Time +} + +// RouteLine is one served name in the list: where it goes, how long its certificate holds, what it last answered. +type RouteLine struct { + Name string `json:"name"` + Proxy string `json:"proxy"` + Reach string `json:"reach"` + Targets []string `json:"targets"` + // Reachable is yes when the last answer upstream gave was below 500, no when it was 500 or more or none came, + // unknown before any request. + Reachable string `json:"reachable"` + // Certificate is its outcome, issuer and expiry in words; ExpiresInDays is the whole days until not-after. + Certificate string `json:"certificate"` + ExpiresInDays *int `json:"expires_in_days,omitempty"` + LastAnswer *routestate.Answer `json:"last_answer,omitempty"` +} + +func (f front) tools() []stdio.Tool { + node := map[string]any{"type": "string", "description": "only the proxy on this machine; every machine when left out"} + return []stdio.Tool{ + {Name: RoutesTool, + Description: "Every site (web name, host name) the mesh's web front serves through route-proxy, the reverse proxy " + + "on the public machine: for each route the module and node it forwards to, its TLS certificate's issuer and " + + "expiry (days left), and the last answer upstream gave (status, time, error), so is X reachable and is its " + + "certificate good is answered without curl. Read from the proxy's routes state on the bus. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}}, + Run: func(args map[string]any) (any, error) { + at, _ := args["node"].(string) + return f.list(at) + }}, + {Name: RouteTool, + Description: "One site the mesh's web front (route-proxy) is asked about, in full: every rule it is served by " + + "(path, module, node, target, refusal, redirect, authentication), its certificate (issuer, not-after, last " + + "renewal and its outcome, or why it failed) and the last upstream answer. A name with no route says so, and " + + "when it was last asked for. Answers is X reachable, why does this site not load. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{ + "name": map[string]any{"type": "string", "description": "the site's name, such as app.example.org"}, + "node": node}}, + Run: func(args map[string]any) (any, error) { + name, _ := args["name"].(string) + at, _ := args["node"].(string) + return f.one(name, at) + }}, + {Name: UnroutedTool, + Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " + + "a TLS handshake that no route serves, newest first, with when it was last asked, by whom, and how often. " + + "A site that should be served and is missing its route shows here. Kept a week. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}}, + Run: func(args map[string]any) (any, error) { + at, _ := args["node"].(string) + return f.refusals(at) + }}, + } +} + +// read is every value of a state, of node's proxy alone when node is named, decoded into a fresh T each. +func read[T any](state reader, node string) ([]T, error) { + keys, err := state.Keys() + if err != nil { + return nil, err + } + var out []T + for _, key := range keys { + at, _ := routestate.Split(key) + if node != "" && at != node { + continue + } + entry, err := state.Get(key) + if err != nil { + return nil, err + } + if entry == nil { + continue // deleted between the list and the read + } + var v T + if err := json.Unmarshal(entry.Value, &v); err != nil { + return nil, fmt.Errorf("%s holds what is not a route-proxy record: %w", key, err) + } + out = append(out, v) + } + return out, nil +} + +func (f front) list(node string) (map[string]any, error) { + routes, err := read[routestate.Route](f.routes, node) + if err != nil { + return nil, err + } + lines := make([]RouteLine, 0, len(routes)) + unreachable, expiring := 0, 0 + for _, r := range routes { + line := f.line(r) + if line.Reachable == "no" { + unreachable++ + } + if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == routestate.Failed { + expiring++ + } + lines = append(lines, line) + } + sort.Slice(lines, func(i, j int) bool { + if lines[i].Name != lines[j].Name { + return lines[i].Name < lines[j].Name + } + return lines[i].Proxy < lines[j].Proxy + }) + return map[string]any{"routes": lines, "served": len(lines), "unreachable": unreachable, + "certificates_failing_or_within_14_days": expiring}, nil +} + +func (f front) line(r routestate.Route) RouteLine { + line := RouteLine{Name: r.Name, Proxy: r.Proxy, Reach: r.Reach, LastAnswer: r.LastAnswer, Reachable: "unknown", + Targets: []string{}} + for _, t := range r.Targets { + line.Targets = append(line.Targets, describe(t)) + } + if a := r.LastAnswer; a != nil { + line.Reachable = "yes" + if a.Error != "" || a.Status >= 500 || a.Status == 0 { + line.Reachable = "no" + } + } + c := r.Certificate + words := []string{} + if c.Outcome != "" { + words = append(words, c.Outcome) + } + if c.Issuer != "" { + words = append(words, "by "+c.Issuer) + } + if c.NotAfter != nil { + days := int(c.NotAfter.Sub(f.now()).Hours() / 24) + line.ExpiresInDays = &days + words = append(words, "until "+c.NotAfter.UTC().Format("2006-01-02")) + } + if c.Error != "" { + words = append(words, "("+c.Error+")") + } + if len(words) == 0 { + words = append(words, "none seen yet") + } + line.Certificate = c.Authority + ": " + strings.Join(words, " ") + return line +} + +// describe is one rule in a few words: path → module@node at target, or what it answers with. +func describe(t routestate.Target) string { + path := t.Path + if path == "" { + path = "/" + } + var what string + switch { + case t.Deny: + what = "refused" + case t.Redirect != "": + what = "redirected to " + t.Redirect + default: + what = fmt.Sprintf("%s@%s at %s", t.Module, t.Node, t.Target) + } + if t.Sealed != "" { + what += " (authentication cannot be checked: " + t.Sealed + ")" + } else if t.Auth { + what += " (authenticated)" + } + return path + " → " + what +} + +func (f front) one(name, node string) (map[string]any, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return nil, fmt.Errorf("name is the site to read, such as app.example.org") + } + routes, err := read[routestate.Route](f.routes, node) + if err != nil { + return nil, err + } + var found []routestate.Route + for _, r := range routes { + if r.Name == name { + found = append(found, r) + } + } + if len(found) > 0 { + out := map[string]any{"name": name, "routed": true, "routes": found} + if len(found) == 1 { + out["reachable"] = f.line(found[0]).Reachable + } + return out, nil + } + asked, err := read[routestate.Unrouted](f.unrouted, node) + if err != nil { + return nil, err + } + for _, a := range asked { + if a.Name == name { + return map[string]any{"name": name, "routed": false, "asked": a, + "says": fmt.Sprintf("no proxy serves %s; the proxy on %s answered \"no route\" for it, last at %s", + name, a.Proxy, a.AskedAt.UTC().Format(time.RFC3339))}, nil + } + } + return map[string]any{"name": name, "routed": false, + "says": fmt.Sprintf("no proxy serves %s, and none was asked for it this week", name)}, nil +} + +func (f front) refusals(node string) (map[string]any, error) { + asked, err := read[routestate.Unrouted](f.unrouted, node) + if err != nil { + return nil, err + } + sort.Slice(asked, func(i, j int) bool { return asked[i].AskedAt.After(asked[j].AskedAt) }) + if asked == nil { + asked = []routestate.Unrouted{} + } + return map[string]any{"unrouted": asked, "names": len(asked)}, nil +} diff --git a/modules/route-proxy/cmd/route-provider/tools_test.go b/modules/route-proxy/cmd/route-provider/tools_test.go new file mode 100644 index 00000000..7161f162 --- /dev/null +++ b/modules/route-proxy/cmd/route-provider/tools_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "sort" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "route-proxy/internal/routestate" +) + +// held is a state as the bus holds it, read-only. +type held map[string]any + +func (h held) Keys() ([]string, error) { + out := []string{} + for k := range h { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (h held) Get(key string) (*stdio.StateEntry, error) { + v, ok := h[key] + if !ok { + return nil, nil + } + raw, err := json.Marshal(v) + return &stdio.StateEntry{Key: key, Value: raw}, err +} + +// The web front's tools answer from the bus, in one call each (issue 774): every site with its target, certificate +// expiry and last answer; one site in full, or that it is not served and when it was asked for; and the names the +// proxy answered "no route for". +func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) { + now := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC) + expires := now.Add(30 * 24 * time.Hour) + routes := held{ + "anchor.grafana.example": routestate.Route{Name: "grafana.example", Proxy: "anchor", Reach: routestate.ReachPublic, + Targets: []routestate.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}}, + Certificate: routestate.Certificate{Authority: routestate.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: routestate.Issued}, + LastAnswer: &routestate.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}}, + "anchor.drive.example": routestate.Route{Name: "drive.example", Proxy: "anchor", Reach: routestate.ReachPublic, + Targets: []routestate.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}}, + Certificate: routestate.Certificate{Authority: routestate.ReachPublic}, + LastAnswer: &routestate.Answer{Status: 200, At: now}}, + } + unrouted := held{ + "anchor.old.example": routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3}, + "anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1}, + } + tools := map[string]stdio.Tool{} + for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() { + tools[tool.Name] = tool + } + + for _, row := range []struct { + tool string + args map[string]any + want []string // what the answer says, in its JSON + }{ + {RoutesTool, nil, []string{`"served":2`, `"unreachable":1`, + `"name":"grafana.example"`, `"reachable":"no"`, `"expires_in_days":30`, `/ → grafana@home-server at http://10.0.0.2:3000`, + `"name":"drive.example"`, `"reachable":"yes"`, `public: none seen yet`}}, + {RoutesTool, map[string]any{"node": "home-server"}, []string{`"served":0`}}, + {RouteTool, map[string]any{"name": "Grafana.Example"}, []string{`"routed":true`, `"reachable":"no"`, + `connection refused`, `"issuer":"Let's Encrypt R11"`}}, + {RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`, + `answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}}, + {RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}}, + {UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`}}, + } { + t.Run(row.tool, func(t *testing.T) { + tool, ok := tools[row.tool] + if !ok { + t.Fatalf("%s is not served", row.tool) + } + if !strings.Contains(tool.Description, "route-proxy") { + t.Errorf("%s's description does not name the web front, so a search does not find it", row.tool) + } + answer, err := tool.Run(row.args) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(answer) + for _, w := range row.want { + if !strings.Contains(string(raw), w) { + t.Errorf("%s(%v) does not say %s:\n%s", row.tool, row.args, w, raw) + } + } + }) + } +} diff --git a/modules/route-proxy/cmd/route-proxy/authority_test.go b/modules/route-proxy/cmd/route-proxy/authority_test.go index 0f70a27d..9c2b3275 100644 --- a/modules/route-proxy/cmd/route-proxy/authority_test.go +++ b/modules/route-proxy/cmd/route-proxy/authority_test.go @@ -19,55 +19,57 @@ const ( // path back: nothing is re-registered, no order reaches the CA, and issuance stops with nothing // saying why. Naming the cache after the authority means the account is only ever found where it is // still valid — the new root lands in a directory with no account in it, and autocert registers. -func TestANewCARootMeansANewAccountCache(t *testing.T) { - const directory = "https://anchor.internal/acme/acme/directory" - before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot)) - after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot)) - if before == after { - t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before) - } -} +func TestTheAccountCacheIsNamedAfterItsAuthority(t *testing.T) { + t.Run("a new CA root means a new account cache", func(t *testing.T) { + const directory = "https://anchor.internal/acme/acme/directory" + before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot)) + after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot)) + if before == after { + t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before) + } + }) -// And the SAME authority keeps the account it registered, restart after restart. -// -// This is the whole reason ACME_CACHE is required in the first place: an account and its -// certificates that did not persist would be re-ordered on every restart, which works silently until -// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the -// mesh writes that file, and a trailing newline coming or going must not throw away an account. -func TestTheSameAuthorityKeepsItsAccount(t *testing.T) { - const directory = "https://anchor.internal/acme/acme/directory" - first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot)) - again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n")) - if first != again { - t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s", - first, again) - } -} + // And the SAME authority keeps the account it registered, restart after restart. + // + // This is the whole reason ACME_CACHE is required in the first place: an account and its + // certificates that did not persist would be re-ordered on every restart, which works silently until + // a rate limit says it does not. Whitespace around the delivered root is not a new authority — the + // mesh writes that file, and a trailing newline coming or going must not throw away an account. + t.Run("the same authority keeps its account", func(t *testing.T) { + const directory = "https://anchor.internal/acme/acme/directory" + first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot)) + again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n")) + if first != again { + t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s", + first, again) + } + }) -// Staging and production are different authorities, and were sharing one account. -// -// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against -// staging reused the staging account, because the cache had no idea they were different. -func TestStagingAndProductionDoNotShareAnAccount(t *testing.T) { - staging := forThisAuthority("/acme", stagingDirectory, nil) - production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil) - if staging == production { - t.Errorf("two issuers share one account: %s", staging) - } -} + // Staging and production are different authorities, and were sharing one account. + // + // The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against + // staging reused the staging account, because the cache had no idea they were different. + t.Run("staging and production do not share an account", func(t *testing.T) { + staging := forThisAuthority("/acme", stagingDirectory, nil) + production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil) + if staging == production { + t.Errorf("two issuers share one account: %s", staging) + } + }) -// It stays inside the directory the mesh gave it, and is a plain name. -// -// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or -// that carried a separator out of the PEM — would put an account somewhere nothing persists. -func TestTheAccountCacheStaysWhereTheMeshPutIt(t *testing.T) { - const cache = "/var/lib/route-proxy/acme" - got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot)) - if !strings.HasPrefix(got, cache+"/") { - t.Fatalf("the account cache is not under %s: %s", cache, got) - } - name := strings.TrimPrefix(got, cache+"/") - if name != filepath.Base(got) || strings.ContainsAny(name, "/.") { - t.Errorf("the account cache is not a plain name: %q", name) - } + // It stays inside the directory the mesh gave it, and is a plain name. + // + // The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or + // that carried a separator out of the PEM — would put an account somewhere nothing persists. + t.Run("the account cache stays where the mesh put it", func(t *testing.T) { + const cache = "/var/lib/route-proxy/acme" + got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot)) + if !strings.HasPrefix(got, cache+"/") { + t.Fatalf("the account cache is not under %s: %s", cache, got) + } + name := strings.TrimPrefix(got, cache+"/") + if name != filepath.Base(got) || strings.ContainsAny(name, "/.") { + t.Errorf("the account cache is not a plain name: %q", name) + } + }) } diff --git a/modules/route-proxy/cmd/route-proxy/bus.go b/modules/route-proxy/cmd/route-proxy/bus.go index 30623add..9dbcc274 100644 --- a/modules/route-proxy/cmd/route-proxy/bus.go +++ b/modules/route-proxy/cmd/route-proxy/bus.go @@ -10,10 +10,23 @@ import ( "github.com/nats-io/nats.go" + "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/process" + + "route-proxy/internal/routestate" ) +// keptStates is every state route-proxy keeps, as both its halves write them: the routes and the refusals this +// proxy writes, readable by others, and the bound entries the provider helper writes. +func keptStates() []announce.State { + return []announce.State{ + {Name: routestate.BoundState}, + {Name: routestate.RoutesState, Readable: true}, + {Name: routestate.UnroutedState, Readable: true}, + } +} + // What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167). // // **The proxy is told, not left to work it out.** Its membership carries the routes it is given — @@ -44,6 +57,14 @@ func followOnce(path string, held *table, fromBus *atomic.Bool) error { if err != nil { return fmt.Errorf("connecting to the bus at %s: %w", membership.BareAddress(account.URL), err) } + // What this proxy keeps, said on the bus (novox/hq ADR 0495; issue 774): the names it serves and the names it + // refused, readable by any module granted a read of them; the provider helper's entries, read by the machines' + // clients through their own grant. Said before the first write, so no write of an unannounced state is refused. + if err := conn.Keeps(keptStates()...); err != nil { + log.Printf("cannot say on the bus what this proxy keeps, so it keeps nothing there: %v", err) + } else { + held.ledger.attach(account.Node, conn.State(routestate.RoutesState), conn.State(routestate.UnroutedState)) + } subject := membership.Subject(account.Node, account.Module) err = conn.FollowMembership(func(issued membership.Membership) { if took := applyMembership(issued, held); took && !fromBus.Swap(true) { diff --git a/modules/route-proxy/cmd/route-proxy/ledger.go b/modules/route-proxy/cmd/route-proxy/ledger.go new file mode 100644 index 00000000..6eb08ed0 --- /dev/null +++ b/modules/route-proxy/cmd/route-proxy/ledger.go @@ -0,0 +1,478 @@ +package main + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "log" + "net" + "net/http" + "reflect" + "strings" + "sync" + "time" + + "golang.org/x/crypto/acme/autocert" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "route-proxy/internal/routestate" +) + +// What this proxy says on the bus about the names it serves and the names it refuses (issue 774). +// +// **On change, never on a timer.** A route arriving or leaving, a certificate issued or failing, an upstream answer +// that differs from the last one written, a name asked for that is not served: each marks its key, and one writer +// puts the marked keys. An answer that is the same as the last one written is written again only once it is older +// than refreshAfter, so a site answering 200 all day costs one write a minute, not one per request. +// +// Until the bus is reached nothing is written; what happened meanwhile is held and written when it is. + +// refreshAfter is how old a written answer, refusal or certificate failure may be before the same one is written +// again with its new time. +const refreshAfter = time.Minute + +// mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes. +const mostAsked = 4096 + +// kvStore is one of the module's states, as process.State is one. +type kvStore interface { + Put(key string, value any) (uint64, error) + Delete(key string) error + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// ledger is what this proxy keeps of what it serves and refuses, and writes to the bus. +type ledger struct { + mu sync.Mutex + now func() time.Time + + node string + routes, unrouted kvStore + + served map[string]*routestate.Route + previous map[string]routestate.Route // what the bus held for this machine when it was reached + asked map[string]*routestate.Unrouted + + answerWritten map[string]time.Time + askedWritten map[string]time.Time + + dirtyRoutes map[string]bool // a name to put when served, to delete when not + dirtyAsked map[string]bool + + kick chan struct{} +} + +func newLedger() *ledger { + return &ledger{ + now: time.Now, + served: map[string]*routestate.Route{}, + previous: map[string]routestate.Route{}, + asked: map[string]*routestate.Unrouted{}, + answerWritten: map[string]time.Time{}, + askedWritten: map[string]time.Time{}, + dirtyRoutes: map[string]bool{}, + dirtyAsked: map[string]bool{}, + kick: make(chan struct{}, 1), + } +} + +// loop writes whatever was marked, each time something was. +func (l *ledger) loop() { + for range l.kick { + l.flush() + } +} + +func (l *ledger) wake() { + select { + case l.kick <- struct{}{}: + default: + } +} + +// attach is the bus reached, as node: what the bus held for this machine is read back, so a restart keeps a name's +// certificate and last answer, and everything held is written. +func (l *ledger) attach(node string, routes, unrouted kvStore) { + if l == nil { + return + } + before := map[string]routestate.Route{} + if keys, err := routes.Keys(); err != nil { + log.Printf("cannot read back what this proxy said it serves: %v", err) + } else { + for _, key := range keys { + at, name := routestate.Split(key) + if at != node { + continue + } + entry, err := routes.Get(key) + if err != nil || entry == nil { + continue + } + var r routestate.Route + if json.Unmarshal(entry.Value, &r) == nil { + before[name] = r + } + } + } + l.mu.Lock() + l.node, l.routes, l.unrouted = node, routes, unrouted + for name, r := range before { + l.previous[name] = r + l.dirtyRoutes[name] = true // put again when served, deleted when no longer + if now := l.served[name]; now != nil { + keepWhatWasKnown(now, r) + } + } + for name := range l.served { + l.dirtyRoutes[name] = true + } + for name := range l.asked { + l.dirtyAsked[name] = true + } + l.mu.Unlock() + l.wake() +} + +// keepWhatWasKnown carries a certificate and a last answer read back from the bus onto a route that knows none yet. +func keepWhatWasKnown(r *routestate.Route, before routestate.Route) { + if r.Certificate.Outcome == "" && r.Certificate.NotAfter == nil && before.Certificate.Authority == r.Certificate.Authority { + r.Certificate = before.Certificate + } + if r.LastAnswer == nil { + r.LastAnswer = before.LastAnswer + } +} + +// serving is the table this proxy now serves: a name new or changed is written, a name gone is deleted, and a name +// that was asked for unserved and now is served leaves the unrouted state. +func (l *ledger) serving(routes map[string][]rule, public map[string]bool) { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + for name, rules := range routes { + reach := routestate.ReachInternal + if public[name] { + reach = routestate.ReachPublic + } + targets := targetsOf(rules) + r := l.served[name] + if r == nil { + r = &routestate.Route{Name: name} + if before, ok := l.previous[name]; ok { + r.Certificate.Authority = reach + keepWhatWasKnown(r, before) + } + l.served[name] = r + l.dirtyRoutes[name] = true + } + if r.Reach != reach || !reflect.DeepEqual(r.Targets, targets) { + r.Reach, r.Targets = reach, targets + l.dirtyRoutes[name] = true + } + if r.Certificate.Authority != reach { + r.Certificate = routestate.Certificate{Authority: reach} + l.dirtyRoutes[name] = true + } + if _, was := l.asked[name]; was { + delete(l.asked, name) + l.dirtyAsked[name] = true + } + } + for name := range l.served { + if _, still := routes[name]; !still { + delete(l.served, name) + delete(l.answerWritten, name) + l.dirtyRoutes[name] = true + } + } + l.wake() +} + +func targetsOf(rules []rule) []routestate.Target { + out := make([]routestate.Target, 0, len(rules)) + for _, r := range rules { + out = append(out, routestate.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target, + Deny: r.policy.deny, Redirect: r.policy.redirectTo, Auth: len(r.policy.users) > 0 || r.policy.sealed != "", + Sealed: r.policy.sealed}) + } + return out +} + +// answered is an upstream answer to a request for name: its status, or the error that stood in for one. +func (l *ledger) answered(name, path string, status int, failed error) { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + r := l.served[name] + if r == nil { + return + } + now := l.now() + why := "" + if failed != nil { + why = failed.Error() + } + before := r.LastAnswer + r.LastAnswer = &routestate.Answer{Status: status, At: now, Error: why, Path: path} + if before != nil && before.Status == status && before.Error == why && now.Sub(l.answerWritten[name]) < refreshAfter { + return + } + l.answerWritten[name] = now + l.dirtyRoutes[name] = true + l.wake() +} + +// refused is a name this proxy answered "no route for": in a request, or in a handshake's server name. +func (l *ledger) refused(name, from string, internalOnly, handshake bool) { + if l == nil || !routestate.Keyable(name) { + return + } + l.mu.Lock() + defer l.mu.Unlock() + if _, isServed := l.served[name]; isServed && !internalOnly { + return + } + now := l.now() + a := l.asked[name] + if a == nil { + if len(l.asked) >= mostAsked { + l.forgetOldestAsked() + } + a = &routestate.Unrouted{Name: name} + l.asked[name] = a + } + if host, _, err := net.SplitHostPort(from); err == nil { + from = host + } + a.AskedAt, a.From, a.InternalOnly, a.Handshake = now, from, internalOnly, handshake + a.Times++ + if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter { + return + } + l.askedWritten[name] = now + l.dirtyAsked[name] = true + l.wake() +} + +// forgetOldestAsked drops the name asked longest ago from memory; the bus keeps it until its time to live ends. +func (l *ledger) forgetOldestAsked() { + oldest := "" + for name, a := range l.asked { + if oldest == "" || a.AskedAt.Before(l.asked[oldest].AskedAt) { + oldest = name + } + } + delete(l.asked, oldest) + delete(l.askedWritten, oldest) + delete(l.dirtyAsked, oldest) +} + +// certified is a certificate an authority issued for name, as autocert stored it. +func (l *ledger) certified(name string, leaf *x509.Certificate) { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + r := l.served[name] + if r == nil { + return + } + now := l.now() + notAfter := leaf.NotAfter.UTC() + r.Certificate.Issuer, r.Certificate.NotAfter = issuerOf(leaf), ¬After + r.Certificate.RenewedAt, r.Certificate.At = &now, &now + r.Certificate.Outcome, r.Certificate.Error = routestate.Issued, "" + l.dirtyRoutes[name] = true + l.wake() +} + +// handedOut is the certificate a handshake for name was answered with: written when it is not the one recorded. +func (l *ledger) handedOut(name string, leaf *x509.Certificate) { + if l == nil || leaf == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + r := l.served[name] + if r == nil { + return + } + notAfter := leaf.NotAfter.UTC() + if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != routestate.Failed { + return + } + now := l.now() + r.Certificate.Issuer, r.Certificate.NotAfter, r.Certificate.At = issuerOf(leaf), ¬After, &now + if r.Certificate.Outcome != routestate.Issued || r.Certificate.Error != "" { + r.Certificate.Outcome, r.Certificate.Error = routestate.Cached, "" + } + l.dirtyRoutes[name] = true + l.wake() +} + +// certificateFailed is a handshake for name that found no certificate to answer with, and why. +func (l *ledger) certificateFailed(name string, failed error) { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + r := l.served[name] + if r == nil { + return + } + now := l.now() + same := r.Certificate.Outcome == routestate.Failed && r.Certificate.Error == failed.Error() + if same && r.Certificate.At != nil && now.Sub(*r.Certificate.At) < refreshAfter { + return + } + r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = routestate.Failed, failed.Error(), &now + l.dirtyRoutes[name] = true + l.wake() +} + +func issuerOf(leaf *x509.Certificate) string { + if cn := leaf.Issuer.CommonName; cn != "" { + if len(leaf.Issuer.Organization) > 0 { + return leaf.Issuer.Organization[0] + " " + cn + } + return cn + } + return leaf.Issuer.String() +} + +// flush writes every marked key. A write that fails stays marked and is tried again at the next change. +func (l *ledger) flush() { + type put struct { + store kvStore + key string + value any // nil deletes + } + l.mu.Lock() + if l.routes == nil { + l.mu.Unlock() + return + } + var puts []put + for name := range l.dirtyRoutes { + p := put{store: l.routes, key: routestate.Key(l.node, name)} + if r := l.served[name]; r != nil { + c := *r + c.Proxy = l.node + p.value = c + } + puts = append(puts, p) + } + for name := range l.dirtyAsked { + p := put{store: l.unrouted, key: routestate.Key(l.node, name)} + if a := l.asked[name]; a != nil { + c := *a + c.Proxy = l.node + p.value = c + } + puts = append(puts, p) + } + l.dirtyRoutes, l.dirtyAsked = map[string]bool{}, map[string]bool{} + l.mu.Unlock() + + for _, p := range puts { + var err error + if p.value == nil { + err = p.store.Delete(p.key) + } else { + _, err = p.store.Put(p.key, p.value) + } + if err != nil { + log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err) + _, name := routestate.Split(p.key) + l.mu.Lock() + if p.store == l.routes { + l.dirtyRoutes[name] = true + } else { + l.dirtyAsked[name] = true + } + l.mu.Unlock() + } + } +} + +// recordingCache is an authority's certificate cache that tells the ledger of every certificate stored in it: the +// one place autocert says it was issued one. +type recordingCache struct { + autocert.Cache + held *table +} + +func (c recordingCache) Put(ctx context.Context, key string, data []byte) error { + if err := c.Cache.Put(ctx, key, data); err != nil { + return err + } + name := strings.TrimSuffix(key, "+rsa") + if strings.Contains(name, "+") || !c.held.routed(name) { + return nil // the account key, a challenge token + } + if leaf := leafOf(data); leaf != nil { + c.held.ledger.certified(bareHost(name), leaf) + } + return nil +} + +// leafOf is the first certificate in what autocert stores: the private key, then the chain, in PEM. +func leafOf(data []byte) *x509.Certificate { + for { + var block *pem.Block + block, data = pem.Decode(data) + if block == nil { + return nil + } + if block.Type == "CERTIFICATE" { + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil + } + return leaf + } + } +} + +// answerRecorder carries a proxied request's answer through and remembers its status and the proxy's error. +type answerRecorder struct { + http.ResponseWriter + status int + err error +} + +func (a *answerRecorder) WriteHeader(status int) { + if a.status == 0 { + a.status = status + } + a.ResponseWriter.WriteHeader(status) +} + +func (a *answerRecorder) Write(b []byte) (int, error) { + if a.status == 0 { + a.status = http.StatusOK + } + return a.ResponseWriter.Write(b) +} + +// Unwrap lets http.ResponseController reach the connection beneath: a flush, an upgrade's hijack. +func (a *answerRecorder) Unwrap() http.ResponseWriter { return a.ResponseWriter } + +// proxyFailed is the reverse proxy's answer when upstream gave none: the error kept for the ledger, and a 502. +func proxyFailed(w http.ResponseWriter, r *http.Request, err error) { + if a, ok := w.(*answerRecorder); ok { + a.err = err + } + log.Printf("http: proxy error for %s%s: %v", r.Host, r.URL.Path, err) + w.WriteHeader(http.StatusBadGateway) +} diff --git a/modules/route-proxy/cmd/route-proxy/ledger_test.go b/modules/route-proxy/cmd/route-proxy/ledger_test.go new file mode 100644 index 00000000..0dee9654 --- /dev/null +++ b/modules/route-proxy/cmd/route-proxy/ledger_test.go @@ -0,0 +1,245 @@ +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "math/big" + "net" + "net/http" + "net/http/httptest" + "sort" + "testing" + "time" + + "golang.org/x/crypto/acme/autocert" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "route-proxy/internal/routestate" +) + +// fakeState is a state on the bus, in memory, counting what was written to it. +type fakeState struct { + values map[string]json.RawMessage + puts int +} + +func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} } + +func (f *fakeState) Put(key string, value any) (uint64, error) { + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + f.values[key] = raw + f.puts++ + return uint64(f.puts), nil +} +func (f *fakeState) Delete(key string) error { delete(f.values, key); return nil } +func (f *fakeState) Keys() ([]string, error) { + out := []string{} + for k := range f.values { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} +func (f *fakeState) Get(key string) (*stdio.StateEntry, error) { + v, ok := f.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: v}, nil +} + +func (f *fakeState) route(t *testing.T, key string) *routestate.Route { + t.Helper() + raw, ok := f.values[key] + if !ok { + return nil + } + var r routestate.Route + if err := json.Unmarshal(raw, &r); err != nil { + t.Fatal(err) + } + return &r +} + +// issuedPEM is what autocert stores for a name: the key, then the certificate, signed by an issuer named here. +func issuedPEM(t *testing.T, name string, notAfter time.Time) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{SerialNumber: big.NewInt(1), DNSNames: []string{name}, + Subject: pkix.Name{CommonName: name}, Issuer: pkix.Name{CommonName: "R11", Organization: []string{"Let's Encrypt"}}, + NotBefore: notAfter.Add(-90 * 24 * time.Hour), NotAfter: notAfter} + issuer := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: template.Issuer, NotBefore: template.NotBefore, + NotAfter: notAfter, IsCA: true, BasicConstraintsValid: true} + der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + keyDER, _ := x509.MarshalECPrivateKey(key) + out := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) +} + +// What the proxy tells the bus (issue 774): each served name with where it goes, its certificate and the last +// upstream answer; each name refused; and only on change. Without the ledger nothing reaches the bus, and the +// operator's question "is grafana reachable" is answered with curl. +func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { + notAfter := time.Date(2027, 1, 28, 12, 0, 0, 0, time.UTC) + gone := func() string { // an address nothing listens on: the upstream that does not answer + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + return l.Addr().String() + }() + _, port, _ := net.SplitHostPort(gone) + contributions := []contribution{ + {From: "grafana", Node: "anchor", At: "127.0.0.1", Values: map[string]any{"name": "grafana.example", "port": mustFloat(t, port)}}, + {From: "drive", Node: "home-server", At: "10.0.0.2", Values: map[string]any{"internal-name": "drive.internal", "port": 8080.0}}, + } + + for _, row := range []struct { + name string + act func(t *testing.T, held *table) + check func(t *testing.T, routes, unrouted *fakeState) + }{ + {"a served name is written with the module and node it forwards to", + func(t *testing.T, held *table) {}, + func(t *testing.T, routes, _ *fakeState) { + r := routes.route(t, "anchor.grafana.example") + if r == nil || r.Proxy != "anchor" || r.Reach != routestate.ReachPublic || len(r.Targets) != 1 || + r.Targets[0].Module != "grafana" || r.Targets[0].Node != "anchor" || r.Targets[0].Target != "http://127.0.0.1:"+port { + t.Fatalf("grafana's route is not on the bus as served: %+v", r) + } + if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != routestate.ReachInternal || + d.Certificate.Authority != routestate.ReachInternal { + t.Fatalf("an internal-only name is not on the bus as internal: %+v", d) + } + }}, + {"a withdrawn route is deleted", + func(t *testing.T, held *table) { + routes, public := routesOf(contributions[1:]) + held.set(routes, public) + }, + func(t *testing.T, routes, _ *fakeState) { + if routes.route(t, "anchor.grafana.example") != nil { + t.Fatal("a route no longer served is still on the bus") + } + }}, + {"an upstream that does not answer is written as a 502 with why, once a minute", + func(t *testing.T, held *table) { + for range 3 { + req := httptest.NewRequest(http.MethodGet, "http://grafana.example/login", nil) + handler(held).ServeHTTP(httptest.NewRecorder(), req) + } + }, + func(t *testing.T, routes, _ *fakeState) { + r := routes.route(t, "anchor.grafana.example") + if r == nil || r.LastAnswer == nil || r.LastAnswer.Status != http.StatusBadGateway || + r.LastAnswer.Error == "" || r.LastAnswer.Path != "/login" { + t.Fatalf("the failed answer is not on the bus: %+v", r) + } + if routes.puts != 3 { // the two routes, then the first answer; the same answer again is not written + t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts) + } + }}, + {"a name not served is written as asked, and leaves when it is served", + func(t *testing.T, held *table) { + req := httptest.NewRequest(http.MethodGet, "http://grafana.other.example/", nil) + req.RemoteAddr = "192.0.2.7:51000" + handler(held).ServeHTTP(httptest.NewRecorder(), req) + held.ledger.flush() + var a routestate.Unrouted + raw := held.ledger.unrouted.(*fakeState).values["anchor.grafana.other.example"] + if err := json.Unmarshal(raw, &a); err != nil || a.From != "192.0.2.7" || a.Times != 1 || a.AskedAt.IsZero() { + t.Fatalf("the refused name is not on the bus as asked: %s", raw) + } + routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor", + Values: map[string]any{"name": "grafana.other.example", "port": 3000.0}})) + held.set(routes, public) + }, + func(t *testing.T, _, unrouted *fakeState) { + if len(unrouted.values) != 0 { + t.Fatalf("a name now served is still written as unrouted: %v", unrouted.values) + } + }}, + {"a handshake for a name not routed is written as asked", + func(t *testing.T, held *table) { + pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return nil, errors.New("no public route for \"forgotten.example\" in this mesh, so no certificate is asked for") + }, nil) + _, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.example"}) + }, + func(t *testing.T, _, unrouted *fakeState) { + var a routestate.Unrouted + if err := json.Unmarshal(unrouted.values["anchor.forgotten.example"], &a); err != nil || !a.Handshake { + t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values) + } + }}, + {"a certificate an authority issued is written with its issuer, expiry and renewal", + func(t *testing.T, held *table) { + cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held} + if err := cache.Put(context.Background(), "grafana.example", issuedPEM(t, "grafana.example", notAfter)); err != nil { + t.Fatal(err) + } + _ = cache.Put(context.Background(), "acme_account+key", []byte("not a certificate")) + }, + func(t *testing.T, routes, _ *fakeState) { + c := routes.route(t, "anchor.grafana.example").Certificate + if c.Outcome != routestate.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil || + !c.NotAfter.Equal(notAfter) || c.RenewedAt == nil { + t.Fatalf("the issued certificate is not on the bus: %+v", c) + } + }}, + {"a certificate that cannot be had is written with why", + func(t *testing.T, held *table) { + pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return nil, errors.New("acme: urn:ietf:params:acme:error:rateLimited") + }, nil) + _, _ = pick(&tls.ClientHelloInfo{ServerName: "grafana.example"}) + }, + func(t *testing.T, routes, _ *fakeState) { + c := routes.route(t, "anchor.grafana.example").Certificate + if c.Outcome != routestate.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" { + t.Fatalf("the failed certificate is not on the bus: %+v", c) + } + }}, + } { + t.Run(row.name, func(t *testing.T) { + held := newTable() + held.ledger = newLedger() + routes, unrouted := newFakeState(), newFakeState() + held.ledger.attach("anchor", routes, unrouted) + table, public := routesOf(contributions) + held.set(table, public) + held.ledger.flush() + row.act(t, held) + held.ledger.flush() + row.check(t, routes, unrouted) + }) + } +} + +func mustFloat(t *testing.T, port string) float64 { + t.Helper() + var n float64 + if err := json.Unmarshal([]byte(port), &n); err != nil { + t.Fatal(err) + } + return n +} diff --git a/modules/route-proxy/cmd/route-proxy/main.go b/modules/route-proxy/cmd/route-proxy/main.go index ebadf638..18c6476d 100644 --- a/modules/route-proxy/cmd/route-proxy/main.go +++ b/modules/route-proxy/cmd/route-proxy/main.go @@ -179,6 +179,8 @@ type rule struct { // registry is the case that needs it — image layers arrive as single requests of gigabytes, and // a proxy's own default refuses them long before the workload is reached. maxRequestBody int64 + // from and node are the consumer that contributed the rule and its machine, which the bus is told (issue 774). + from, node string } // table is what the proxy is currently serving, replaced whole whenever the file changes. @@ -202,6 +204,8 @@ type table struct { // machine's address on the private network, as the mesh issued it in this proxy's membership // (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside. inside sources + // ledger is what this proxy says on the bus about what it serves and refuses (issue 774); nil says nothing. + ledger *ledger } // sources is who may be served an internal name: the private network's addresses as the mesh @@ -289,6 +293,7 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) { t.to = made t.public = public t.mu.Unlock() + t.ledger.serving(made, public) } // inOrder puts the rules for one host into the order they are matched in, and does so totally. @@ -437,6 +442,8 @@ func run() error { } held := newTable() + held.ledger = newLedger() + go held.ledger.loop() // **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries // the routes and who the mesh is; the file carries the routes alone, so while the proxy reads // it an internal name is served to this machine and to nobody else — refused, never opened. @@ -487,7 +494,7 @@ func run() error { "to persist, or every restart orders them again") } publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")), - onlyWhatTheMeshSaid(held)) + onlyWhatTheMeshSaid(held), held) if err != nil { return err } @@ -499,7 +506,7 @@ func run() error { var internalManager *autocert.Manager if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" { internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")), - onlyInternalNamesTheMeshSaid(held)) + onlyInternalNamesTheMeshSaid(held), held) if err != nil { return fmt.Errorf("internal certificate authority: %w", err) } @@ -619,7 +626,7 @@ func (p *probedResponse) replayTo(w http.ResponseWriter) { // one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS // with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public // issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead. -func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) { +func newManager(cache, directory, bundle string, policy autocert.HostPolicy, held *table) (*autocert.Manager, error) { client := &acme.Client{DirectoryURL: directory} var root []byte if bundle != "" { @@ -651,7 +658,7 @@ func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*a // different names because their directories differ. mine := forThisAuthority(cache, directory, root) return &autocert.Manager{ - Cache: autocert.DirCache(mine), + Cache: recordingCache{Cache: autocert.DirCache(mine), held: held}, Prompt: autocert.AcceptTOS, HostPolicy: policy, Client: client, @@ -700,9 +707,10 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer if internalManager != nil { fromInternal = internalManager.TLSConfig().GetCertificate } - return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { if held.eligibleForInternalACME(hello.ServerName) { if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) { + held.ledger.refused(bareHost(hello.ServerName), hello.Conn.RemoteAddr().String(), true, true) return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", hello.ServerName) } @@ -712,6 +720,32 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer } return fromPublic(hello) } + // What the bus is told of a handshake (issue 774): a routed name's certificate, or why it has none; a name not + // routed here, which a browser asking for a site the mesh forgot to route meets before any request is made. + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := pick(hello) + name := bareHost(hello.ServerName) + switch { + case name == "": + case !held.routed(name): + from := "" + if hello.Conn != nil { + from = hello.Conn.RemoteAddr().String() + } + held.ledger.refused(name, from, false, true) + case err != nil: + if !strings.HasPrefix(err.Error(), "no public route for") { + held.ledger.certificateFailed(name, err) + } + case cert != nil: + leaf := cert.Leaf + if leaf == nil && len(cert.Certificate) > 0 { + leaf, _ = x509.ParseCertificate(cert.Certificate[0]) + } + held.ledger.handedOut(name, leaf) + } + return cert, err + } } // newTable is an empty routing table. @@ -737,6 +771,9 @@ func handler(held *table) http.Handler { // this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the // jail's filter expects it. log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr) + if hidden || !held.routed(r.Host) { + held.ledger.refused(bareHost(r.Host), r.RemoteAddr, hidden, false) + } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusNotFound) if !hidden && held.routed(r.Host) { @@ -786,7 +823,14 @@ func handler(held *table) http.Handler { r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody) } - matched.to.ServeHTTP(w, r) + // The upstream's answer is what the bus is told of this name (issue 774): its status, or why there was none. + answer := &answerRecorder{ResponseWriter: w} + matched.to.ServeHTTP(answer, r) + status := answer.status + if status == 0 && answer.err == nil && r.Header.Get("Upgrade") != "" { + status = http.StatusSwitchingProtocols + } + held.ledger.answered(bareHost(r.Host), r.URL.Path, status, answer.err) }) } @@ -955,6 +999,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) } made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port) } + made.from, made.node = c.From, c.Node if name != "" { host := strings.ToLower(name) @@ -1072,5 +1117,5 @@ func towards(where *url.URL) *httputil.ReverseProxy { pr.SetURL(where) pr.Out.Host = pr.In.Host pr.SetXForwarded() - }} + }, ErrorHandler: proxyFailed} } diff --git a/modules/route-proxy/cmd/route-proxy/routes_test.go b/modules/route-proxy/cmd/route-proxy/routes_test.go index 1ad9a263..a8447f45 100644 --- a/modules/route-proxy/cmd/route-proxy/routes_test.go +++ b/modules/route-proxy/cmd/route-proxy/routes_test.go @@ -121,19 +121,6 @@ func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) { } } -// A route with neither name has nothing to be served under, and is still skipped. -func TestARouteWithNeitherNameIsSkipped(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ - {"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}} - ]}`)) - if err != nil { - t.Fatal(err) - } - if len(routes) != 0 || len(public) != 0 { - t.Errorf("a route that named nothing was served: %v %v", routes, public) - } -} - // A route with no internal-name composed gets no second host — the ordinary case, unchanged. func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { routes, _, err := routesFrom(write(t, `{"given":[ @@ -161,21 +148,6 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { } } -// Skipped rather than served wrongly. A route with no port would proxy to :0. -func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ - {"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}}, - {"from":"b","node":"n","at":"n.internal","values":{"port":8080}}, - {"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}} - ]}`)) - if err != nil { - t.Fatal(err) - } - if len(routes) != 1 || targetOf(routes, "fine.example") == "" { - t.Fatalf("an unusable contribution was served: %v", routes) - } -} - // A route may name a target reached over https, for a backend that terminates its own TLS — the // shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise. func TestARouteMayTargetHttps(t *testing.T) { @@ -194,19 +166,6 @@ func TestARouteMayTargetHttps(t *testing.T) { } } -// A scheme that is neither http nor https is refused rather than guessed at. -func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ - {"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}} - ]}`)) - if err != nil { - t.Fatal(err) - } - if len(routes) != 0 { - t.Fatalf("a route with an unusable scheme was served: %v", routes) - } -} - // End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still // reached when the route declared `insecure`, and the response comes back through unmodified. func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) { @@ -455,23 +414,6 @@ func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) { } } -// A limit that is not a whole positive number of bytes takes the route with it. Serving it without -// the limit would carry exactly what the module said not to carry, and report success doing it. -func TestARouteWithAnUnusableLimitIsSkipped(t *testing.T) { - for _, asked := range []string{`"lots"`, `-1`, `0`, `1.5`} { - routes, _, err := routesFrom(write(t, `{"given":[ - {"from":"registry","node":"anchor","at":"anchor.internal", - "values":{"name":"images.example","port":5000,"max-request-body":`+asked+`}} - ]}`)) - if err != nil { - t.Fatal(err) - } - if len(routes["images.example"]) != 0 { - t.Errorf("a route asking for a max-request-body of %s was served anyway: %v", asked, routes) - } - } -} - // A request larger than the route carries is refused by the proxy, with the limit named, and the // workload never sees it. A request within it is proxied normally. func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) { @@ -532,3 +474,32 @@ func post(url, host, body string) (*http.Response, error) { asked.Header.Set("Content-Type", "application/octet-stream") return http.DefaultClient.Do(asked) } + +// A contribution that cannot be served is skipped rather than served wrongly: one that names nothing to be served +// under; one with no port, which would proxy to :0; a scheme that is neither http nor https, refused rather than +// guessed at; and a body limit that is not a whole positive number of bytes, since serving it without the limit would +// carry exactly what the module said not to carry, and report success doing it. +func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) { + for _, row := range []struct{ why, given, served string }{ + {"neither name", `{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}`, ""}, + {"no port", `{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}}, + {"from":"b","node":"n","at":"n.internal","values":{"port":8080}}, + {"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}`, "fine.example"}, + {"an unknown scheme", `{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}`, ""}, + {"a limit in words", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":"lots"}}`, ""}, + {"a negative limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":-1}}`, ""}, + {"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""}, + {"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""}, + } { + routes, public, err := routesFrom(write(t, `{"given":[`+row.given+`]}`)) + if err != nil { + t.Fatal(err) + } + switch { + case row.served == "" && (len(routes) != 0 || len(public) != 0): + t.Errorf("%s: an unusable contribution was served: %v %v", row.why, routes, public) + case row.served != "" && (len(routes) != 1 || targetOf(routes, row.served) == ""): + t.Errorf("%s: an unusable contribution was served beside the usable one: %v", row.why, routes) + } + } +} diff --git a/modules/route-proxy/internal/routestate/routestate.go b/modules/route-proxy/internal/routestate/routestate.go new file mode 100644 index 00000000..239d4cc9 --- /dev/null +++ b/modules/route-proxy/internal/routestate/routestate.go @@ -0,0 +1,122 @@ +// Package routestate is what the web front keeps on the bus about the names it serves, and about the names it was asked +// for and does not serve (issue 774): the shape both halves of route-proxy agree on. The proxy writes it, on change, +// in its states `routes` and `unrouted`, announced readable; the module's tools read it from there, so the question +// "is this site reachable, and is its certificate good" is answered from the bus rather than with curl. +// +// A key is `.`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds +// no dot, so the first dot splits them. +package routestate + +import ( + "regexp" + "strings" + "time" +) + +// The states route-proxy keeps, by their local names (the buckets are route-proxy_). +const ( + // RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got. + RoutesState = "routes" + // UnroutedState holds one key per name the proxy answered "no route for" with, and when it was last asked. + UnroutedState = "unrouted" + // BoundState is the provider helper's bucket of consumer entries (mesh-sdk go/provider). + BoundState = "bound" +) + +// The words a route's reach and a certificate's authority are said in. +const ( + ReachPublic = "public" + ReachInternal = "internal" +) + +// The words a certificate's outcome is said in. +const ( + // Issued: the authority issued it while this proxy ran, at RenewedAt. + Issued = "issued" + // Cached: this proxy served it from what it kept, and saw no issuance of it since it started. + Cached = "cached" + // Failed: the last attempt to have one failed, Error says why. + Failed = "failed" +) + +// Target is one rule a name is served by: what a request matching it reaches, or what it is answered with. +type Target struct { + // Path is the path prefix the rule is scoped to; empty is every path. + Path string `json:"path,omitempty"` + // Module and Node are the consumer that contributed the route and the machine it runs on. + Module string `json:"module,omitempty"` + Node string `json:"node,omitempty"` + // Target is where the request is forwarded, scheme://address:port; empty for a rule that answers itself. + Target string `json:"target,omitempty"` + Deny bool `json:"deny,omitempty"` + Redirect string `json:"redirect,omitempty"` + // Auth says the rule asks for credentials; Sealed says why it cannot check them, and so refuses everything. + Auth bool `json:"auth,omitempty"` + Sealed string `json:"sealed,omitempty"` +} + +// Certificate is what this proxy knows of a name's certificate. +type Certificate struct { + // Authority is which issuer certifies the name: public (the ACME directory the module names) or internal (the + // mesh's own). + Authority string `json:"authority"` + Issuer string `json:"issuer,omitempty"` + NotAfter *time.Time `json:"not_after,omitempty"` + // RenewedAt is when an authority last issued it to this proxy. + RenewedAt *time.Time `json:"renewed_at,omitempty"` + // Outcome is issued, cached or failed; empty until a handshake or an issuance said anything. + Outcome string `json:"outcome,omitempty"` + Error string `json:"error,omitempty"` + At *time.Time `json:"at,omitempty"` +} + +// Answer is the last answer the proxy got from upstream for a name: the status it carried back, or the error that +// stood in for one (a 502). +type Answer struct { + Status int `json:"status"` + At time.Time `json:"at"` + Error string `json:"error,omitempty"` + Path string `json:"path,omitempty"` +} + +// Route is one served name, the value of a key of RoutesState. +type Route struct { + Name string `json:"name"` + // Proxy is the machine whose proxy serves it. + Proxy string `json:"proxy"` + // Reach is public (a route's own name) or internal (only its internal-name, served inside the private network). + Reach string `json:"reach"` + Targets []Target `json:"targets"` + Certificate Certificate `json:"certificate"` + LastAnswer *Answer `json:"last_answer,omitempty"` +} + +// Unrouted is one name the proxy answered "no route for", the value of a key of UnroutedState. +type Unrouted struct { + Name string `json:"name"` + Proxy string `json:"proxy"` + // AskedAt is when the name was last asked for; From is who asked last. + AskedAt time.Time `json:"asked_at"` + From string `json:"from,omitempty"` + // Times is how often it was asked since the proxy last started. + Times int `json:"times"` + // InternalOnly says the name is served, but only inside the private network, and was asked from outside. + InternalOnly bool `json:"internal_only,omitempty"` + // Handshake says it was asked in a TLS handshake (the name a client sent), not in an HTTP request. + Handshake bool `json:"handshake,omitempty"` +} + +var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`) + +// Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request +// is whatever a client sent; one no key could hold is no site anybody runs. +func Keyable(name string) bool { return keyable.MatchString(name) && !strings.Contains(name, "..") } + +// Key is the key of name served or asked at node's proxy. +func Key(node, name string) string { return node + "." + name } + +// Split is a key's node and name. +func Split(key string) (node, name string) { + node, name, _ = strings.Cut(key, ".") + return node, name +} diff --git a/modules/route-proxy/module.json b/modules/route-proxy/module.json index 4a527ea5..620a733d 100644 --- a/modules/route-proxy/module.json +++ b/modules/route-proxy/module.json @@ -244,8 +244,29 @@ } ], "state": [ - "bound" + "bound", + "routes", + { + "name": "unrouted", + "ttl-seconds": 604800 + } ], + "tools": [ + "route_proxy_routes", + "route_proxy_route", + "route_proxy_unrouted" + ], + "replaces": { + "route_proxy_routes": [ + "curl -I" + ], + "route_proxy_route": [ + "openssl s_client -connect" + ], + "route_proxy_unrouted": [ + "docker logs route-proxy" + ] + }, "reads": [ "mesh-controller.assignments" ]