diff --git a/modules/route-proxy/README.md b/modules/route-proxy/README.md index 15f5e5f2..2767eca7 100644 --- a/modules/route-proxy/README.md +++ b/modules/route-proxy/README.md @@ -98,7 +98,7 @@ The proxy keeps two states, announced readable (`conn.Keeps`, novox/hq ADR 0495) | state | one key per | holds | written when | |---|---|---|---| | `routes` | served name | the rules it is served by (path, module, node, target, refusal, redirect, authentication); its certificate (authority, issuer, not-after, last renewal, outcome, or why it failed); the last upstream answer (status, time, error, path) | a route arrives, changes or leaves; an authority stores a certificate; a handshake fails to find one or hands out one not yet recorded; an upstream answer differs from the last one written, or the same one is a minute old | -| `unrouted` | name answered "no route for", in a request or a TLS handshake | when it was last asked, by whom, how often since the start, whether it is an internal-only name asked from outside | first asked, then at most once a minute; deleted when the name is served; kept a week | +| `unrouted` | name answered "no route for", in a request or a TLS handshake, under a domain the proxy serves | when it was last asked, how often since the start, whether it is an internal-only name asked from outside; never who asked (the journal keeps that, for fail2ban) | first asked, at most five new names a minute, then at most once a minute; deleted when the name is served; kept a week. Every other refusal is only counted, in one key per proxy (`._other`: the last minute's count and the total), written once a minute. A refused write is dropped, never retried | Nothing is written on a timer. The provider helper's binary serves the module's tools from those states, one call each: `route_proxy_routes` (every served name with its target, certificate expiry and last answer), `route_proxy_route` diff --git a/modules/route-proxy/cmd/route-provider/tools.go b/modules/route-proxy/cmd/route-provider/tools.go index 5855fd93..c01a0ba4 100644 --- a/modules/route-proxy/cmd/route-provider/tools.go +++ b/modules/route-proxy/cmd/route-provider/tools.go @@ -79,8 +79,10 @@ func (f front) tools() []stdio.Tool { }}, {Name: UnroutedTool, Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " + - "a TLS handshake that no route serves, newest first, with when it was last asked, by whom, and how often. " + - "A site that should be served and is missing its route shows here. Kept a week. Read-only.", + "a TLS handshake that no route serves, newest first, with when it was last asked and how often. Only names under a " + + "domain the proxy serves are kept by name, a few new ones a minute; every other refusal (a scan) is counted per " + + "proxy, the last minute's count and the total since its start. A site that should be served and is missing " + + "its route shows here. Kept a week. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}}, Run: func(args map[string]any) (any, error) { at, _ := args["node"].(string) @@ -224,7 +226,7 @@ func (f front) one(name, node string) (map[string]any, error) { } return out, nil } - asked, err := read[routestate.Unrouted](f.unrouted, node) + asked, _, err := f.asked(node) if err != nil { return nil, err } @@ -240,7 +242,7 @@ func (f front) one(name, node string) (map[string]any, error) { } func (f front) refusals(node string) (map[string]any, error) { - asked, err := read[routestate.Unrouted](f.unrouted, node) + asked, others, err := f.asked(node) if err != nil { return nil, err } @@ -248,5 +250,47 @@ func (f front) refusals(node string) (map[string]any, error) { if asked == nil { asked = []routestate.Unrouted{} } - return map[string]any{"unrouted": asked, "names": len(asked)}, nil + return map[string]any{"unrouted": asked, "names": len(asked), "other_refusals": others}, nil +} + +// asked is the names refused and kept by name, and each proxy's count of the refusals not kept by name. +func (f front) asked(node string) ([]routestate.Unrouted, []routestate.Other, error) { + keys, err := f.unrouted.Keys() + if err != nil { + return nil, nil, err + } + names, others := keySet{}, keySet{} + for _, key := range keys { + if _, name := routestate.Split(key); name == routestate.OtherName { + others[key] = true + } else { + names[key] = true + } + } + asked, err := read[routestate.Unrouted](only{f.unrouted, names}, node) + if err != nil { + return nil, nil, err + } + counted, err := read[routestate.Other](only{f.unrouted, others}, node) + if counted == nil { + counted = []routestate.Other{} + } + return asked, counted, err +} + +// keySet is a set of keys; only is a state narrowed to them. +type keySet map[string]bool + +type only struct { + reader + keys keySet +} + +func (o only) Keys() ([]string, error) { + out := make([]string, 0, len(o.keys)) + for k := range o.keys { + out = append(out, k) + } + sort.Strings(out) + return out, nil } diff --git a/modules/route-proxy/cmd/route-provider/tools_test.go b/modules/route-proxy/cmd/route-provider/tools_test.go index 7161f162..3194e755 100644 --- a/modules/route-proxy/cmd/route-provider/tools_test.go +++ b/modules/route-proxy/cmd/route-provider/tools_test.go @@ -52,6 +52,7 @@ func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) { unrouted := held{ "anchor.old.example": routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3}, "anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1}, + "anchor._other": routestate.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900}, } tools := map[string]stdio.Tool{} for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() { @@ -72,7 +73,8 @@ func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) { {RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`, `answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}}, {RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}}, - {UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`}}, + {UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`, + `"other_refusals":[{"proxy":"anchor"`, `"last_minute":40`, `"since_start":900`}}, } { t.Run(row.tool, func(t *testing.T) { tool, ok := tools[row.tool] diff --git a/modules/route-proxy/cmd/route-proxy/ledger.go b/modules/route-proxy/cmd/route-proxy/ledger.go index 6eb08ed0..fce37132 100644 --- a/modules/route-proxy/cmd/route-proxy/ledger.go +++ b/modules/route-proxy/cmd/route-proxy/ledger.go @@ -6,9 +6,9 @@ import ( "encoding/json" "encoding/pem" "log" - "net" "net/http" "reflect" + "sort" "strings" "sync" "time" @@ -36,6 +36,9 @@ const refreshAfter = time.Minute // mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes. const mostAsked = 4096 +// newPerMinute is how many names not seen before are kept by name in a minute; past it they are only counted. +const newPerMinute = 5 + // kvStore is one of the module's states, as process.State is one. type kvStore interface { Put(key string, value any) (uint64, error) @@ -62,6 +65,16 @@ type ledger struct { dirtyRoutes map[string]bool // a name to put when served, to delete when not dirtyAsked map[string]bool + // domains are the registrable domains of the names served: a refused name under one is kept by name. + domains map[string]bool + newMinute time.Time + newInMinute int + other routestate.Other + otherMinute time.Time + otherInMinute int + otherDirty bool + routesFailing, askFailed bool // a failed write said once, until one succeeds + kick chan struct{} } @@ -75,6 +88,7 @@ func newLedger() *ledger { askedWritten: map[string]time.Time{}, dirtyRoutes: map[string]bool{}, dirtyAsked: map[string]bool{}, + domains: map[string]bool{}, kick: make(chan struct{}, 1), } } @@ -184,6 +198,12 @@ func (l *ledger) serving(routes map[string][]rule, public map[string]bool) { l.dirtyAsked[name] = true } } + l.domains = map[string]bool{} + for name := range routes { + if d := routestate.Registrable(name); d != "" { + l.domains[d] = true + } + } for name := range l.served { if _, still := routes[name]; !still { delete(l.served, name) @@ -231,8 +251,13 @@ func (l *ledger) answered(name, path string, status int, failed error) { } // refused is a name this proxy answered "no route for": in a request, or in a handshake's server name. -func (l *ledger) refused(name, from string, internalOnly, handshake bool) { - if l == nil || !routestate.Keyable(name) { +// +// **Only a name under a domain this proxy serves is kept by name**, and at most newPerMinute new ones a minute: a scan +// sends any name it likes, and each kept name is a key on the bus. Every other refusal is counted, and the count is +// written once a minute under the proxy's OtherName key. Nothing here scans: forgetting old names is the writer's. +// Who asked is not kept: the journal line beside it is what the machine's intrusion prevention reads. +func (l *ledger) refused(name string, internalOnly, handshake bool) { + if l == nil { return } l.mu.Lock() @@ -243,16 +268,14 @@ func (l *ledger) refused(name, from string, internalOnly, handshake bool) { now := l.now() a := l.asked[name] if a == nil { - if len(l.asked) >= mostAsked { - l.forgetOldestAsked() + if !routestate.Keyable(name) || !l.domains[routestate.Registrable(name)] || !l.mayAddName(now) { + l.countOther(now) + return } a = &routestate.Unrouted{Name: name} l.asked[name] = a } - if host, _, err := net.SplitHostPort(from); err == nil { - from = host - } - a.AskedAt, a.From, a.InternalOnly, a.Handshake = now, from, internalOnly, handshake + a.AskedAt, a.InternalOnly, a.Handshake = now, internalOnly, handshake a.Times++ if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter { return @@ -262,17 +285,49 @@ func (l *ledger) refused(name, from string, internalOnly, handshake bool) { l.wake() } -// forgetOldestAsked drops the name asked longest ago from memory; the bus keeps it until its time to live ends. +// mayAddName says whether another new name may be kept this minute, and counts it. +func (l *ledger) mayAddName(now time.Time) bool { + if minute := now.Truncate(time.Minute); !minute.Equal(l.newMinute) { + l.newMinute, l.newInMinute = minute, 0 + } + if l.newInMinute >= newPerMinute { + return false + } + l.newInMinute++ + return true +} + +// countOther counts a refusal not kept by name. A minute's count is written when the next minute's first one comes. +func (l *ledger) countOther(now time.Time) { + minute := now.Truncate(time.Minute) + if !minute.Equal(l.otherMinute) { + if l.otherInMinute > 0 { + l.other.Minute, l.other.LastMinute = l.otherMinute, l.otherInMinute + l.otherDirty = true + l.wake() + } + l.otherMinute, l.otherInMinute = minute, 0 + } + l.otherInMinute++ + l.other.SinceStart++ +} + +// forgetOldestAsked drops the names asked longest ago from memory until mostAsked are left; the bus keeps them until +// their time to live ends. Called by the writer, never on a request. func (l *ledger) forgetOldestAsked() { - oldest := "" - for name, a := range l.asked { - if oldest == "" || a.AskedAt.Before(l.asked[oldest].AskedAt) { - oldest = name - } + if len(l.asked) <= mostAsked { + return + } + names := make([]string, 0, len(l.asked)) + for name := range l.asked { + names = append(names, name) + } + sort.Slice(names, func(i, j int) bool { return l.asked[names[i]].AskedAt.Before(l.asked[names[j]].AskedAt) }) + for _, name := range names[:len(names)-mostAsked] { + delete(l.asked, name) + delete(l.askedWritten, name) + delete(l.dirtyAsked, name) } - delete(l.asked, oldest) - delete(l.askedWritten, oldest) - delete(l.dirtyAsked, oldest) } // certified is a certificate an authority issued for name, as autocert stored it. @@ -350,21 +405,25 @@ func issuerOf(leaf *x509.Certificate) string { return leaf.Issuer.String() } -// flush writes every marked key. A write that fails stays marked and is tried again at the next change. +// flush writes every marked key. A route's write that fails stays marked and is tried again at the next change; a +// refused name's write that fails is dropped, never tried again: the bucket refusing it is full, or the bus is away, +// and either way it is not worth a storm of retries. Each kind of failure is logged once, until a write succeeds. func (l *ledger) flush() { type put struct { store kvStore key string value any // nil deletes + route bool } l.mu.Lock() if l.routes == nil { l.mu.Unlock() return } + l.forgetOldestAsked() var puts []put for name := range l.dirtyRoutes { - p := put{store: l.routes, key: routestate.Key(l.node, name)} + p := put{store: l.routes, key: routestate.Key(l.node, name), route: true} if r := l.served[name]; r != nil { c := *r c.Proxy = l.node @@ -381,7 +440,12 @@ func (l *ledger) flush() { } puts = append(puts, p) } - l.dirtyRoutes, l.dirtyAsked = map[string]bool{}, map[string]bool{} + if l.otherDirty { + o := l.other + o.Proxy = l.node + puts = append(puts, put{store: l.unrouted, key: routestate.Key(l.node, routestate.OtherName), value: o}) + } + l.dirtyRoutes, l.dirtyAsked, l.otherDirty = map[string]bool{}, map[string]bool{}, false l.mu.Unlock() for _, p := range puts { @@ -391,17 +455,27 @@ func (l *ledger) flush() { } else { _, err = p.store.Put(p.key, p.value) } - if err != nil { - log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err) - _, name := routestate.Split(p.key) - l.mu.Lock() - if p.store == l.routes { - l.dirtyRoutes[name] = true - } else { - l.dirtyAsked[name] = true + l.mu.Lock() + switch { + case err == nil && p.route: + l.routesFailing = false + case err == nil: + l.askFailed = false + case p.route: + if !l.routesFailing { + log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err) } - l.mu.Unlock() + l.routesFailing = true + _, name := routestate.Split(p.key) + l.dirtyRoutes[name] = true + default: + if !l.askFailed { + log.Printf("cannot write %s to the bus, dropped with every refused name until a write succeeds: %v", + p.key, err) + } + l.askFailed = true } + l.mu.Unlock() } } diff --git a/modules/route-proxy/cmd/route-proxy/ledger_test.go b/modules/route-proxy/cmd/route-proxy/ledger_test.go index 0dee9654..fba0dd6c 100644 --- a/modules/route-proxy/cmd/route-proxy/ledger_test.go +++ b/modules/route-proxy/cmd/route-proxy/ledger_test.go @@ -11,11 +11,13 @@ import ( "encoding/json" "encoding/pem" "errors" + "fmt" "math/big" "net" "net/http" "net/http/httptest" "sort" + "strings" "testing" "time" @@ -30,11 +32,17 @@ import ( type fakeState struct { values map[string]json.RawMessage puts int + tries int // every Put asked, refused or not + fail error // what every Put is refused with, while set } func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} } func (f *fakeState) Put(key string, value any) (uint64, error) { + f.tries++ + if f.fail != nil { + return 0, f.fail + } raw, err := json.Marshal(value) if err != nil { return 0, err @@ -158,19 +166,22 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts) } }}, - {"a name not served is written as asked, and leaves when it is served", + {"a name under a served domain is written as asked, without who asked, and leaves when it is served", func(t *testing.T, held *table) { - req := httptest.NewRequest(http.MethodGet, "http://grafana.other.example/", nil) + req := httptest.NewRequest(http.MethodGet, "http://wiki.grafana.example/", nil) req.RemoteAddr = "192.0.2.7:51000" handler(held).ServeHTTP(httptest.NewRecorder(), req) held.ledger.flush() var a routestate.Unrouted - raw := held.ledger.unrouted.(*fakeState).values["anchor.grafana.other.example"] - if err := json.Unmarshal(raw, &a); err != nil || a.From != "192.0.2.7" || a.Times != 1 || a.AskedAt.IsZero() { + raw := held.ledger.unrouted.(*fakeState).values["anchor.wiki.grafana.example"] + if err := json.Unmarshal(raw, &a); err != nil || a.Times != 1 || a.AskedAt.IsZero() { t.Fatalf("the refused name is not on the bus as asked: %s", raw) } + if strings.Contains(string(raw), "192.0.2.7") || strings.Contains(string(raw), `"from"`) { + t.Fatalf("who asked is on the bus: %s", raw) + } routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor", - Values: map[string]any{"name": "grafana.other.example", "port": 3000.0}})) + Values: map[string]any{"name": "wiki.grafana.example", "port": 3000.0}})) held.set(routes, public) }, func(t *testing.T, _, unrouted *fakeState) { @@ -181,16 +192,51 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { {"a handshake for a name not routed is written as asked", func(t *testing.T, held *table) { pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - return nil, errors.New("no public route for \"forgotten.example\" in this mesh, so no certificate is asked for") + return nil, errors.New("no public route for \"forgotten.grafana.example\" in this mesh, so no certificate is asked for") }, nil) - _, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.example"}) + _, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.grafana.example"}) }, func(t *testing.T, _, unrouted *fakeState) { var a routestate.Unrouted - if err := json.Unmarshal(unrouted.values["anchor.forgotten.example"], &a); err != nil || !a.Handshake { + if err := json.Unmarshal(unrouted.values["anchor.forgotten.grafana.example"], &a); err != nil || !a.Handshake { t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values) } }}, + {"a flood of names writes at most the limit by name, counts the rest once a minute, and never retries a refused write", + func(t *testing.T, held *table) { + clock := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC) + held.ledger.now = func() time.Time { return clock } + full := held.ledger.unrouted.(*fakeState) + for i := range 40 { + held.ledger.refused(fmt.Sprintf("x%d.grafana.example", i), false, false) // under a served domain + held.ledger.refused(fmt.Sprintf("x%d.scan.test", i), false, true) // under none + } + held.ledger.flush() + if named := len(full.values); named != newPerMinute { + t.Fatalf("a flood wrote %d names, want the limit %d: %v", named, newPerMinute, full.values) + } + full.fail = errors.New("nats: maximum bytes exceeded") + clock = clock.Add(time.Minute) + held.ledger.refused("y.grafana.example", false, false) // a new minute: one more name, + held.ledger.refused("v.scan.test", false, false) // and the last minute's count + held.ledger.flush() + tries := full.tries + held.ledger.refused("y.grafana.example", false, false) + held.ledger.refused("z.scan.test", false, false) + held.ledger.flush() + if full.tries != tries { + t.Fatalf("writes the bucket refused were tried again: %d tries, then %d", tries, full.tries) + } + full.fail = nil + clock = clock.Add(time.Minute) + held.ledger.refused("w.scan.test", false, false) + }, + func(t *testing.T, _, unrouted *fakeState) { + var o routestate.Other + if err := json.Unmarshal(unrouted.values["anchor._other"], &o); err != nil || o.LastMinute != 2 || o.SinceStart != 78 { + t.Fatalf("the refusals not kept by name are not counted on the bus: %s", unrouted.values["anchor._other"]) + } + }}, {"a certificate an authority issued is written with its issuer, expiry and renewal", func(t *testing.T, held *table) { cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held} diff --git a/modules/route-proxy/cmd/route-proxy/main.go b/modules/route-proxy/cmd/route-proxy/main.go index 18c6476d..5288f55f 100644 --- a/modules/route-proxy/cmd/route-proxy/main.go +++ b/modules/route-proxy/cmd/route-proxy/main.go @@ -710,7 +710,7 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { if held.eligibleForInternalACME(hello.ServerName) { if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) { - held.ledger.refused(bareHost(hello.ServerName), hello.Conn.RemoteAddr().String(), true, true) + held.ledger.refused(bareHost(hello.ServerName), true, true) return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", hello.ServerName) } @@ -728,11 +728,7 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer switch { case name == "": case !held.routed(name): - from := "" - if hello.Conn != nil { - from = hello.Conn.RemoteAddr().String() - } - held.ledger.refused(name, from, false, true) + held.ledger.refused(name, false, true) case err != nil: if !strings.HasPrefix(err.Error(), "no public route for") { held.ledger.certificateFailed(name, err) @@ -772,7 +768,7 @@ func handler(held *table) http.Handler { // jail's filter expects it. log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr) if hidden || !held.routed(r.Host) { - held.ledger.refused(bareHost(r.Host), r.RemoteAddr, hidden, false) + held.ledger.refused(bareHost(r.Host), hidden, false) } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusNotFound) diff --git a/modules/route-proxy/internal/routestate/routestate.go b/modules/route-proxy/internal/routestate/routestate.go index 239d4cc9..44dd82d8 100644 --- a/modules/route-proxy/internal/routestate/routestate.go +++ b/modules/route-proxy/internal/routestate/routestate.go @@ -8,6 +8,7 @@ package routestate import ( + "net" "regexp" "strings" "time" @@ -95,9 +96,9 @@ type Route struct { type Unrouted struct { Name string `json:"name"` Proxy string `json:"proxy"` - // AskedAt is when the name was last asked for; From is who asked last. + // AskedAt is when the name was last asked for. Who asked is not kept here: the journal keeps it, for the + // machine's intrusion prevention. AskedAt time.Time `json:"asked_at"` - From string `json:"from,omitempty"` // Times is how often it was asked since the proxy last started. Times int `json:"times"` // InternalOnly says the name is served, but only inside the private network, and was asked from outside. @@ -106,6 +107,34 @@ type Unrouted struct { Handshake bool `json:"handshake,omitempty"` } +// OtherName is the name of the one key per proxy that counts the refusals not kept by name: names under no domain +// the proxy serves (what a scan sends), and names past the limit of new names a minute. Its key is +// `._other`; no host name begins with an underscore. +const OtherName = "_other" + +// Other is the value of a proxy's OtherName key. +type Other struct { + Proxy string `json:"proxy"` + // Minute is the last whole minute counted, and LastMinute how many refusals it held. + Minute time.Time `json:"minute"` + LastMinute int `json:"last_minute"` + // SinceStart is every refusal not kept by name since the proxy started. + SinceStart int `json:"since_start"` +} + +// Registrable is the domain a name is registered under, as its last two labels; "" for an address or a single +// label. Two labels is wrong for a public suffix of two (co.uk); it only decides which refused names are kept by name. +func Registrable(name string) string { + if net.ParseIP(strings.Trim(name, "[]")) != nil { + return "" + } + labels := strings.Split(name, ".") + if len(labels) < 2 { + return "" + } + return strings.Join(labels[len(labels)-2:], ".") +} + var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`) // Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request