Keep a scan out of route-proxy's unrouted state: only names under a served domain, a few a minute, no visitor address (issue 774)

Review of ddae6729: a scanner's random Host or SNI names each became a key, bounded only by the
bucket's lifetime and size, and every failed write was retried at each change and logged each time.

A refused name is now kept by name only under the registrable domain of a name the proxy serves, and
at most five new names a minute; every other refusal is counted in memory and written once a minute
under <node>._other (last minute, since start), which route_proxy_unrouted shows. A refused write of
that state is dropped and logged once until a write succeeds; trimming memory to 4096 names moved
from the request to the writer. The visitor's address is no longer kept: the journal has it.

Quick: drop unrouted keys from strangers by a fixed allow-list. Thorough: bound by the proxy's own
domains, by rate, and by never retrying. Chose thorough: the scan is constant on a public machine.

modules/route-proxy/cmd/route-provider/tools.go

@@ -79,8 +79,10 @@ func (f front) tools() []stdio.Tool {
 			}},
 		{Name: UnroutedTool,
 			Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " +
-				"a TLS handshake that no route serves, newest first, with when it was last asked, by whom, and how often. " +
-				"A site that should be served and is missing its route shows here. Kept a week. Read-only.",
+				"a TLS handshake that no route serves, newest first, with when it was last asked and how often. Only names under a " +
+				"domain the proxy serves are kept by name, a few new ones a minute; every other refusal (a scan) is counted per " +
+				"proxy, the last minute's count and the total since its start. A site that should be served and is missing " +
+				"its route shows here. Kept a week. Read-only.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
 			Run: func(args map[string]any) (any, error) {
 				at, _ := args["node"].(string)
@@ -224,7 +226,7 @@ func (f front) one(name, node string) (map[string]any, error) {
 		}
 		return out, nil
 	}
-	asked, err := read[routestate.Unrouted](f.unrouted, node)
+	asked, _, err := f.asked(node)
 	if err != nil {
 		return nil, err
 	}
@@ -240,7 +242,7 @@ func (f front) one(name, node string) (map[string]any, error) {
 }
 
 func (f front) refusals(node string) (map[string]any, error) {
-	asked, err := read[routestate.Unrouted](f.unrouted, node)
+	asked, others, err := f.asked(node)
 	if err != nil {
 		return nil, err
 	}
@@ -248,5 +250,47 @@ func (f front) refusals(node string) (map[string]any, error) {
 	if asked == nil {
 		asked = []routestate.Unrouted{}
 	}
-	return map[string]any{"unrouted": asked, "names": len(asked)}, nil
+	return map[string]any{"unrouted": asked, "names": len(asked), "other_refusals": others}, nil
+}
+
+// asked is the names refused and kept by name, and each proxy's count of the refusals not kept by name.
+func (f front) asked(node string) ([]routestate.Unrouted, []routestate.Other, error) {
+	keys, err := f.unrouted.Keys()
+	if err != nil {
+		return nil, nil, err
+	}
+	names, others := keySet{}, keySet{}
+	for _, key := range keys {
+		if _, name := routestate.Split(key); name == routestate.OtherName {
+			others[key] = true
+		} else {
+			names[key] = true
+		}
+	}
+	asked, err := read[routestate.Unrouted](only{f.unrouted, names}, node)
+	if err != nil {
+		return nil, nil, err
+	}
+	counted, err := read[routestate.Other](only{f.unrouted, others}, node)
+	if counted == nil {
+		counted = []routestate.Other{}
+	}
+	return asked, counted, err
+}
+
+// keySet is a set of keys; only is a state narrowed to them.
+type keySet map[string]bool
+
+type only struct {
+	reader
+	keys keySet
+}
+
+func (o only) Keys() ([]string, error) {
+	out := make([]string, 0, len(o.keys))
+	for k := range o.keys {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out, nil
 }

modules/route-proxy/cmd/route-provider/tools_test.go

@@ -52,6 +52,7 @@ func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) {
 	unrouted := held{
 		"anchor.old.example":  routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3},
 		"anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1},
+		"anchor._other":       routestate.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900},
 	}
 	tools := map[string]stdio.Tool{}
 	for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() {
@@ -72,7 +73,8 @@ func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) {
 		{RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`,
 			`answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}},
 		{RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}},
-		{UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`}},
+		{UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`,
+			`"other_refusals":[{"proxy":"anchor"`, `"last_minute":40`, `"since_start":900`}},
 	} {
 		t.Run(row.tool, func(t *testing.T) {
 			tool, ok := tools[row.tool]

modules/route-proxy/cmd/route-proxy/ledger.go

@@ -6,9 +6,9 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"log"
-	"net"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -36,6 +36,9 @@ const refreshAfter = time.Minute
 // mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes.
 const mostAsked = 4096
 
+// newPerMinute is how many names not seen before are kept by name in a minute; past it they are only counted.
+const newPerMinute = 5
+
 // kvStore is one of the module's states, as process.State is one.
 type kvStore interface {
 	Put(key string, value any) (uint64, error)
@@ -62,6 +65,16 @@ type ledger struct {
 	dirtyRoutes map[string]bool // a name to put when served, to delete when not
 	dirtyAsked  map[string]bool
 
+	// domains are the registrable domains of the names served: a refused name under one is kept by name.
+	domains                  map[string]bool
+	newMinute                time.Time
+	newInMinute              int
+	other                    routestate.Other
+	otherMinute              time.Time
+	otherInMinute            int
+	otherDirty               bool
+	routesFailing, askFailed bool // a failed write said once, until one succeeds
+
 	kick chan struct{}
 }
 
@@ -75,6 +88,7 @@ func newLedger() *ledger {
 		askedWritten:  map[string]time.Time{},
 		dirtyRoutes:   map[string]bool{},
 		dirtyAsked:    map[string]bool{},
+		domains:       map[string]bool{},
 		kick:          make(chan struct{}, 1),
 	}
 }
@@ -184,6 +198,12 @@ func (l *ledger) serving(routes map[string][]rule, public map[string]bool) {
 			l.dirtyAsked[name] = true
 		}
 	}
+	l.domains = map[string]bool{}
+	for name := range routes {
+		if d := routestate.Registrable(name); d != "" {
+			l.domains[d] = true
+		}
+	}
 	for name := range l.served {
 		if _, still := routes[name]; !still {
 			delete(l.served, name)
@@ -231,8 +251,13 @@ func (l *ledger) answered(name, path string, status int, failed error) {
 }
 
 // refused is a name this proxy answered "no route for": in a request, or in a handshake's server name.
-func (l *ledger) refused(name, from string, internalOnly, handshake bool) {
-	if l == nil || !routestate.Keyable(name) {
+//
+// **Only a name under a domain this proxy serves is kept by name**, and at most newPerMinute new ones a minute: a scan
+// sends any name it likes, and each kept name is a key on the bus. Every other refusal is counted, and the count is
+// written once a minute under the proxy's OtherName key. Nothing here scans: forgetting old names is the writer's.
+// Who asked is not kept: the journal line beside it is what the machine's intrusion prevention reads.
+func (l *ledger) refused(name string, internalOnly, handshake bool) {
+	if l == nil {
 		return
 	}
 	l.mu.Lock()
@@ -243,16 +268,14 @@ func (l *ledger) refused(name, from string, internalOnly, handshake bool) {
 	now := l.now()
 	a := l.asked[name]
 	if a == nil {
-		if len(l.asked) >= mostAsked {
-			l.forgetOldestAsked()
+		if !routestate.Keyable(name) || !l.domains[routestate.Registrable(name)] || !l.mayAddName(now) {
+			l.countOther(now)
+			return
 		}
 		a = &routestate.Unrouted{Name: name}
 		l.asked[name] = a
 	}
-	if host, _, err := net.SplitHostPort(from); err == nil {
-		from = host
-	}
-	a.AskedAt, a.From, a.InternalOnly, a.Handshake = now, from, internalOnly, handshake
+	a.AskedAt, a.InternalOnly, a.Handshake = now, internalOnly, handshake
 	a.Times++
 	if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter {
 		return
@@ -262,17 +285,49 @@ func (l *ledger) refused(name, from string, internalOnly, handshake bool) {
 	l.wake()
 }
 
-// forgetOldestAsked drops the name asked longest ago from memory; the bus keeps it until its time to live ends.
+// mayAddName says whether another new name may be kept this minute, and counts it.
+func (l *ledger) mayAddName(now time.Time) bool {
+	if minute := now.Truncate(time.Minute); !minute.Equal(l.newMinute) {
+		l.newMinute, l.newInMinute = minute, 0
+	}
+	if l.newInMinute >= newPerMinute {
+		return false
+	}
+	l.newInMinute++
+	return true
+}
+
+// countOther counts a refusal not kept by name. A minute's count is written when the next minute's first one comes.
+func (l *ledger) countOther(now time.Time) {
+	minute := now.Truncate(time.Minute)
+	if !minute.Equal(l.otherMinute) {
+		if l.otherInMinute > 0 {
+			l.other.Minute, l.other.LastMinute = l.otherMinute, l.otherInMinute
+			l.otherDirty = true
+			l.wake()
+		}
+		l.otherMinute, l.otherInMinute = minute, 0
+	}
+	l.otherInMinute++
+	l.other.SinceStart++
+}
+
+// forgetOldestAsked drops the names asked longest ago from memory until mostAsked are left; the bus keeps them until
+// their time to live ends. Called by the writer, never on a request.
 func (l *ledger) forgetOldestAsked() {
-	oldest := ""
-	for name, a := range l.asked {
-		if oldest == "" || a.AskedAt.Before(l.asked[oldest].AskedAt) {
-			oldest = name
-		}
+	if len(l.asked) <= mostAsked {
+		return
+	}
+	names := make([]string, 0, len(l.asked))
+	for name := range l.asked {
+		names = append(names, name)
+	}
+	sort.Slice(names, func(i, j int) bool { return l.asked[names[i]].AskedAt.Before(l.asked[names[j]].AskedAt) })
+	for _, name := range names[:len(names)-mostAsked] {
+		delete(l.asked, name)
+		delete(l.askedWritten, name)
+		delete(l.dirtyAsked, name)
 	}
-	delete(l.asked, oldest)
-	delete(l.askedWritten, oldest)
-	delete(l.dirtyAsked, oldest)
 }
 
 // certified is a certificate an authority issued for name, as autocert stored it.
@@ -350,21 +405,25 @@ func issuerOf(leaf *x509.Certificate) string {
 	return leaf.Issuer.String()
 }
 
-// flush writes every marked key. A write that fails stays marked and is tried again at the next change.
+// flush writes every marked key. A route's write that fails stays marked and is tried again at the next change; a
+// refused name's write that fails is dropped, never tried again: the bucket refusing it is full, or the bus is away,
+// and either way it is not worth a storm of retries. Each kind of failure is logged once, until a write succeeds.
 func (l *ledger) flush() {
 	type put struct {
 		store kvStore
 		key   string
 		value any // nil deletes
+		route bool
 	}
 	l.mu.Lock()
 	if l.routes == nil {
 		l.mu.Unlock()
 		return
 	}
+	l.forgetOldestAsked()
 	var puts []put
 	for name := range l.dirtyRoutes {
-		p := put{store: l.routes, key: routestate.Key(l.node, name)}
+		p := put{store: l.routes, key: routestate.Key(l.node, name), route: true}
 		if r := l.served[name]; r != nil {
 			c := *r
 			c.Proxy = l.node
@@ -381,7 +440,12 @@ func (l *ledger) flush() {
 		}
 		puts = append(puts, p)
 	}
-	l.dirtyRoutes, l.dirtyAsked = map[string]bool{}, map[string]bool{}
+	if l.otherDirty {
+		o := l.other
+		o.Proxy = l.node
+		puts = append(puts, put{store: l.unrouted, key: routestate.Key(l.node, routestate.OtherName), value: o})
+	}
+	l.dirtyRoutes, l.dirtyAsked, l.otherDirty = map[string]bool{}, map[string]bool{}, false
 	l.mu.Unlock()
 
 	for _, p := range puts {
@@ -391,17 +455,27 @@ func (l *ledger) flush() {
 		} else {
 			_, err = p.store.Put(p.key, p.value)
 		}
-		if err != nil {
-			log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err)
-			_, name := routestate.Split(p.key)
-			l.mu.Lock()
-			if p.store == l.routes {
-				l.dirtyRoutes[name] = true
-			} else {
-				l.dirtyAsked[name] = true
+		l.mu.Lock()
+		switch {
+		case err == nil && p.route:
+			l.routesFailing = false
+		case err == nil:
+			l.askFailed = false
+		case p.route:
+			if !l.routesFailing {
+				log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err)
 			}
-			l.mu.Unlock()
+			l.routesFailing = true
+			_, name := routestate.Split(p.key)
+			l.dirtyRoutes[name] = true
+		default:
+			if !l.askFailed {
+				log.Printf("cannot write %s to the bus, dropped with every refused name until a write succeeds: %v",
+					p.key, err)
+			}
+			l.askFailed = true
 		}
+		l.mu.Unlock()
 	}
 }
 

modules/route-proxy/cmd/route-proxy/ledger_test.go

@@ -11,11 +11,13 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"errors"
+	"fmt"
 	"math/big"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -30,11 +32,17 @@ import (
 type fakeState struct {
 	values map[string]json.RawMessage
 	puts   int
+	tries  int   // every Put asked, refused or not
+	fail   error // what every Put is refused with, while set
 }
 
 func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} }
 
 func (f *fakeState) Put(key string, value any) (uint64, error) {
+	f.tries++
+	if f.fail != nil {
+		return 0, f.fail
+	}
 	raw, err := json.Marshal(value)
 	if err != nil {
 		return 0, err
@@ -158,19 +166,22 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) {
 					t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts)
 				}
 			}},
-		{"a name not served is written as asked, and leaves when it is served",
+		{"a name under a served domain is written as asked, without who asked, and leaves when it is served",
 			func(t *testing.T, held *table) {
-				req := httptest.NewRequest(http.MethodGet, "http://grafana.other.example/", nil)
+				req := httptest.NewRequest(http.MethodGet, "http://wiki.grafana.example/", nil)
 				req.RemoteAddr = "192.0.2.7:51000"
 				handler(held).ServeHTTP(httptest.NewRecorder(), req)
 				held.ledger.flush()
 				var a routestate.Unrouted
-				raw := held.ledger.unrouted.(*fakeState).values["anchor.grafana.other.example"]
-				if err := json.Unmarshal(raw, &a); err != nil || a.From != "192.0.2.7" || a.Times != 1 || a.AskedAt.IsZero() {
+				raw := held.ledger.unrouted.(*fakeState).values["anchor.wiki.grafana.example"]
+				if err := json.Unmarshal(raw, &a); err != nil || a.Times != 1 || a.AskedAt.IsZero() {
 					t.Fatalf("the refused name is not on the bus as asked: %s", raw)
 				}
+				if strings.Contains(string(raw), "192.0.2.7") || strings.Contains(string(raw), `"from"`) {
+					t.Fatalf("who asked is on the bus: %s", raw)
+				}
 				routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor",
-					Values: map[string]any{"name": "grafana.other.example", "port": 3000.0}}))
+					Values: map[string]any{"name": "wiki.grafana.example", "port": 3000.0}}))
 				held.set(routes, public)
 			},
 			func(t *testing.T, _, unrouted *fakeState) {
@@ -181,16 +192,51 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) {
 		{"a handshake for a name not routed is written as asked",
 			func(t *testing.T, held *table) {
 				pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
-					return nil, errors.New("no public route for \"forgotten.example\" in this mesh, so no certificate is asked for")
+					return nil, errors.New("no public route for \"forgotten.grafana.example\" in this mesh, so no certificate is asked for")
 				}, nil)
-				_, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.example"})
+				_, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.grafana.example"})
 			},
 			func(t *testing.T, _, unrouted *fakeState) {
 				var a routestate.Unrouted
-				if err := json.Unmarshal(unrouted.values["anchor.forgotten.example"], &a); err != nil || !a.Handshake {
+				if err := json.Unmarshal(unrouted.values["anchor.forgotten.grafana.example"], &a); err != nil || !a.Handshake {
 					t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values)
 				}
 			}},
+		{"a flood of names writes at most the limit by name, counts the rest once a minute, and never retries a refused write",
+			func(t *testing.T, held *table) {
+				clock := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
+				held.ledger.now = func() time.Time { return clock }
+				full := held.ledger.unrouted.(*fakeState)
+				for i := range 40 {
+					held.ledger.refused(fmt.Sprintf("x%d.grafana.example", i), false, false) // under a served domain
+					held.ledger.refused(fmt.Sprintf("x%d.scan.test", i), false, true)        // under none
+				}
+				held.ledger.flush()
+				if named := len(full.values); named != newPerMinute {
+					t.Fatalf("a flood wrote %d names, want the limit %d: %v", named, newPerMinute, full.values)
+				}
+				full.fail = errors.New("nats: maximum bytes exceeded")
+				clock = clock.Add(time.Minute)
+				held.ledger.refused("y.grafana.example", false, false) // a new minute: one more name,
+				held.ledger.refused("v.scan.test", false, false)       // and the last minute's count
+				held.ledger.flush()
+				tries := full.tries
+				held.ledger.refused("y.grafana.example", false, false)
+				held.ledger.refused("z.scan.test", false, false)
+				held.ledger.flush()
+				if full.tries != tries {
+					t.Fatalf("writes the bucket refused were tried again: %d tries, then %d", tries, full.tries)
+				}
+				full.fail = nil
+				clock = clock.Add(time.Minute)
+				held.ledger.refused("w.scan.test", false, false)
+			},
+			func(t *testing.T, _, unrouted *fakeState) {
+				var o routestate.Other
+				if err := json.Unmarshal(unrouted.values["anchor._other"], &o); err != nil || o.LastMinute != 2 || o.SinceStart != 78 {
+					t.Fatalf("the refusals not kept by name are not counted on the bus: %s", unrouted.values["anchor._other"])
+				}
+			}},
 		{"a certificate an authority issued is written with its issuer, expiry and renewal",
 			func(t *testing.T, held *table) {
 				cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held}

modules/route-proxy/cmd/route-proxy/main.go

@@ -710,7 +710,7 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
 	pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
 		if held.eligibleForInternalACME(hello.ServerName) {
 			if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
-				held.ledger.refused(bareHost(hello.ServerName), hello.Conn.RemoteAddr().String(), true, true)
+				held.ledger.refused(bareHost(hello.ServerName), true, true)
 				return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
 					hello.ServerName)
 			}
@@ -728,11 +728,7 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
 		switch {
 		case name == "":
 		case !held.routed(name):
-			from := ""
-			if hello.Conn != nil {
-				from = hello.Conn.RemoteAddr().String()
-			}
-			held.ledger.refused(name, from, false, true)
+			held.ledger.refused(name, false, true)
 		case err != nil:
 			if !strings.HasPrefix(err.Error(), "no public route for") {
 				held.ledger.certificateFailed(name, err)
@@ -772,7 +768,7 @@ func handler(held *table) http.Handler {
 			// jail's filter expects it.
 			log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
 			if hidden || !held.routed(r.Host) {
-				held.ledger.refused(bareHost(r.Host), r.RemoteAddr, hidden, false)
+				held.ledger.refused(bareHost(r.Host), hidden, false)
 			}
 			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 			w.WriteHeader(http.StatusNotFound)