Give route-proxy its state on the bus and three tools, so a site's route, certificate and last answer are read from the mesh instead of with curl (issue 774)

The web front kept nothing on the bus: whether grafana or drive was routed, where to, with which
certificate, and what it last answered could only be learned with curl or the container's log.

The proxy now keeps `routes` and `unrouted` (announced readable, keyed <node>.<name>), written on
change by one writer: route set changes, certificates stored by autocert's cache or failing at a
handshake, upstream answers that differ from the last written (or are a minute old), and names
refused in a request or a handshake. The provider helper's bundle serves route_proxy_routes,
route_proxy_route and route_proxy_unrouted from those states.

Quick: log the routes at start and grep the journal. Thorough: the state on the bus and the tools
that read it, mesh-wide, in one call. Chose thorough: the operator's question is mesh-wide and
recurs, and the log says nothing about certificates or answers.

Tests: two table tests added (the ledger, the tools), both failing without the change; four
authority tests and four skipped-route tests merged into two, so the module has four fewer.

modules/route-proxy/README.md

@@ -90,3 +90,17 @@ A small Go program beside the proxy (`cmd/route-provider`) runs the SDK's provid
 and the offer says `"answered-by": "helper"`: a handed consumer (gitea's origins file) is filled by its machine's client
 from this module's entry, which carries the public names the controller composed for it. The module is **not** marked
 moved: its own files, `acme.env` and the broker secret, are composed as before, and the proxy itself is unchanged.
+
+## What it says on the bus, and its tools (issue 774)
+
+The proxy keeps two states, announced readable (`conn.Keeps`, novox/hq ADR 0495), each keyed `<node>.<name>`:
+
+| state | one key per | holds | written when |
+|---|---|---|---|
+| `routes` | served name | the rules it is served by (path, module, node, target, refusal, redirect, authentication); its certificate (authority, issuer, not-after, last renewal, outcome, or why it failed); the last upstream answer (status, time, error, path) | a route arrives, changes or leaves; an authority stores a certificate; a handshake fails to find one or hands out one not yet recorded; an upstream answer differs from the last one written, or the same one is a minute old |
+| `unrouted` | name answered "no route for", in a request or a TLS handshake | when it was last asked, by whom, how often since the start, whether it is an internal-only name asked from outside | first asked, then at most once a minute; deleted when the name is served; kept a week |
+
+Nothing is written on a timer. The provider helper's binary serves the module's tools from those states, one call each:
+`route_proxy_routes` (every served name with its target, certificate expiry and last answer), `route_proxy_route`
+(one name in full, or that it is not served and when it was last asked for) and `route_proxy_unrouted` (the names
+refused, newest first).

modules/route-proxy/cmd/route-provider/main.go

@@ -1,6 +1,6 @@
 // route-provider: route-proxy's Go half, one binary the node's runtime launches and speaks MCP to over stdio
-// through the Go SDK. It serves no tools; it runs the SDK's provider helper that answers the consumers of `route`
+// through the Go SDK. It runs the SDK's provider helper that answers the consumers of `route` (issue 792), and serves
-// (issue 792). The TypeScript adapter beside it keeps writing the proxy's files.
+// the module's tools, which read what the proxy keeps on the bus (issue 774, tools.go).
 //
 // stdout is the MCP channel, so everything this module says, it says on stderr.
 package main
@@ -9,9 +9,12 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
 	"git.novox.be/novox/mesh-sdk/go/provider"
+
+	"route-proxy/internal/routestate"
 )
 
 func say(format string, args ...any) {
@@ -30,7 +33,9 @@ func main() {
 			}
 		}()
 	}
-	if err := stdio.Serve("", nil); err != nil {
+	tools := front{routes: stdio.State(routestate.RoutesState), unrouted: stdio.State(routestate.UnroutedState),
+		now: time.Now}.tools()
+	if err := stdio.Serve("", tools); err != nil {
 		say("%v", err)
 		os.Exit(1)
 	}

modules/route-proxy/cmd/route-provider/provider_test.go

@@ -95,7 +95,8 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) {
 	if len(provides) != 1 || provides[0].(map[string]any)["answered-by"] != "helper" {
 		t.Fatalf("route is not said to be answered by the helper: %v", provides)
 	}
-	for _, k := range []string{"build", "provides", "state", "reads"} {
+	// Its states and its tools are what it says on the bus, not what it runs or writes (issue 774).
+	for _, k := range []string{"build", "provides", "state", "reads", "tools", "replaces"} {
 		delete(m, k)
 	}
 	pinned, err := json.Marshal(m) // sorted keys, compact

modules/route-proxy/cmd/route-provider/tools.go

@@ -0,0 +1,252 @@
+package main
+
+// route-proxy's tools (issue 774): the web front's command line. Each answers from what the proxy keeps on the bus
+// (route-proxy_routes and route-proxy_unrouted, written on change by the proxy itself), in one call, for every
+// machine running a proxy: which sites are served and where they go, whether their certificates are good, what the
+// last request through each got back, and which names were asked for and are served nowhere.
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"route-proxy/internal/routestate"
+)
+
+// The tools' names, as the manifest declares them.
+const (
+	RoutesTool   = "route_proxy_routes"
+	RouteTool    = "route_proxy_route"
+	UnroutedTool = "route_proxy_unrouted"
+)
+
+// reader is one of the module's states, as stdio.KeptState reads it.
+type reader interface {
+	Keys() ([]string, error)
+	Get(key string) (*stdio.StateEntry, error)
+}
+
+// front is what the tools read: the routes and the refusals, and the clock a certificate's days left are counted by.
+type front struct {
+	routes, unrouted reader
+	now              func() time.Time
+}
+
+// RouteLine is one served name in the list: where it goes, how long its certificate holds, what it last answered.
+type RouteLine struct {
+	Name    string   `json:"name"`
+	Proxy   string   `json:"proxy"`
+	Reach   string   `json:"reach"`
+	Targets []string `json:"targets"`
+	// Reachable is yes when the last answer upstream gave was below 500, no when it was 500 or more or none came,
+	// unknown before any request.
+	Reachable string `json:"reachable"`
+	// Certificate is its outcome, issuer and expiry in words; ExpiresInDays is the whole days until not-after.
+	Certificate   string             `json:"certificate"`
+	ExpiresInDays *int               `json:"expires_in_days,omitempty"`
+	LastAnswer    *routestate.Answer `json:"last_answer,omitempty"`
+}
+
+func (f front) tools() []stdio.Tool {
+	node := map[string]any{"type": "string", "description": "only the proxy on this machine; every machine when left out"}
+	return []stdio.Tool{
+		{Name: RoutesTool,
+			Description: "Every site (web name, host name) the mesh's web front serves through route-proxy, the reverse proxy " +
+				"on the public machine: for each route the module and node it forwards to, its TLS certificate's issuer and " +
+				"expiry (days left), and the last answer upstream gave (status, time, error), so is X reachable and is its " +
+				"certificate good is answered without curl. Read from the proxy's routes state on the bus. Read-only.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
+			Run: func(args map[string]any) (any, error) {
+				at, _ := args["node"].(string)
+				return f.list(at)
+			}},
+		{Name: RouteTool,
+			Description: "One site the mesh's web front (route-proxy) is asked about, in full: every rule it is served by " +
+				"(path, module, node, target, refusal, redirect, authentication), its certificate (issuer, not-after, last " +
+				"renewal and its outcome, or why it failed) and the last upstream answer. A name with no route says so, and " +
+				"when it was last asked for. Answers is X reachable, why does this site not load. Read-only.",
+			Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
+				"name": map[string]any{"type": "string", "description": "the site's name, such as app.example.org"},
+				"node": node}},
+			Run: func(args map[string]any) (any, error) {
+				name, _ := args["name"].(string)
+				at, _ := args["node"].(string)
+				return f.one(name, at)
+			}},
+		{Name: UnroutedTool,
+			Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " +
+				"a TLS handshake that no route serves, newest first, with when it was last asked, by whom, and how often. " +
+				"A site that should be served and is missing its route shows here. Kept a week. Read-only.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
+			Run: func(args map[string]any) (any, error) {
+				at, _ := args["node"].(string)
+				return f.refusals(at)
+			}},
+	}
+}
+
+// read is every value of a state, of node's proxy alone when node is named, decoded into a fresh T each.
+func read[T any](state reader, node string) ([]T, error) {
+	keys, err := state.Keys()
+	if err != nil {
+		return nil, err
+	}
+	var out []T
+	for _, key := range keys {
+		at, _ := routestate.Split(key)
+		if node != "" && at != node {
+			continue
+		}
+		entry, err := state.Get(key)
+		if err != nil {
+			return nil, err
+		}
+		if entry == nil {
+			continue // deleted between the list and the read
+		}
+		var v T
+		if err := json.Unmarshal(entry.Value, &v); err != nil {
+			return nil, fmt.Errorf("%s holds what is not a route-proxy record: %w", key, err)
+		}
+		out = append(out, v)
+	}
+	return out, nil
+}
+
+func (f front) list(node string) (map[string]any, error) {
+	routes, err := read[routestate.Route](f.routes, node)
+	if err != nil {
+		return nil, err
+	}
+	lines := make([]RouteLine, 0, len(routes))
+	unreachable, expiring := 0, 0
+	for _, r := range routes {
+		line := f.line(r)
+		if line.Reachable == "no" {
+			unreachable++
+		}
+		if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == routestate.Failed {
+			expiring++
+		}
+		lines = append(lines, line)
+	}
+	sort.Slice(lines, func(i, j int) bool {
+		if lines[i].Name != lines[j].Name {
+			return lines[i].Name < lines[j].Name
+		}
+		return lines[i].Proxy < lines[j].Proxy
+	})
+	return map[string]any{"routes": lines, "served": len(lines), "unreachable": unreachable,
+		"certificates_failing_or_within_14_days": expiring}, nil
+}
+
+func (f front) line(r routestate.Route) RouteLine {
+	line := RouteLine{Name: r.Name, Proxy: r.Proxy, Reach: r.Reach, LastAnswer: r.LastAnswer, Reachable: "unknown",
+		Targets: []string{}}
+	for _, t := range r.Targets {
+		line.Targets = append(line.Targets, describe(t))
+	}
+	if a := r.LastAnswer; a != nil {
+		line.Reachable = "yes"
+		if a.Error != "" || a.Status >= 500 || a.Status == 0 {
+			line.Reachable = "no"
+		}
+	}
+	c := r.Certificate
+	words := []string{}
+	if c.Outcome != "" {
+		words = append(words, c.Outcome)
+	}
+	if c.Issuer != "" {
+		words = append(words, "by "+c.Issuer)
+	}
+	if c.NotAfter != nil {
+		days := int(c.NotAfter.Sub(f.now()).Hours() / 24)
+		line.ExpiresInDays = &days
+		words = append(words, "until "+c.NotAfter.UTC().Format("2006-01-02"))
+	}
+	if c.Error != "" {
+		words = append(words, "("+c.Error+")")
+	}
+	if len(words) == 0 {
+		words = append(words, "none seen yet")
+	}
+	line.Certificate = c.Authority + ": " + strings.Join(words, " ")
+	return line
+}
+
+// describe is one rule in a few words: path → module@node at target, or what it answers with.
+func describe(t routestate.Target) string {
+	path := t.Path
+	if path == "" {
+		path = "/"
+	}
+	var what string
+	switch {
+	case t.Deny:
+		what = "refused"
+	case t.Redirect != "":
+		what = "redirected to " + t.Redirect
+	default:
+		what = fmt.Sprintf("%s@%s at %s", t.Module, t.Node, t.Target)
+	}
+	if t.Sealed != "" {
+		what += " (authentication cannot be checked: " + t.Sealed + ")"
+	} else if t.Auth {
+		what += " (authenticated)"
+	}
+	return path + " → " + what
+}
+
+func (f front) one(name, node string) (map[string]any, error) {
+	name = strings.ToLower(strings.TrimSpace(name))
+	if name == "" {
+		return nil, fmt.Errorf("name is the site to read, such as app.example.org")
+	}
+	routes, err := read[routestate.Route](f.routes, node)
+	if err != nil {
+		return nil, err
+	}
+	var found []routestate.Route
+	for _, r := range routes {
+		if r.Name == name {
+			found = append(found, r)
+		}
+	}
+	if len(found) > 0 {
+		out := map[string]any{"name": name, "routed": true, "routes": found}
+		if len(found) == 1 {
+			out["reachable"] = f.line(found[0]).Reachable
+		}
+		return out, nil
+	}
+	asked, err := read[routestate.Unrouted](f.unrouted, node)
+	if err != nil {
+		return nil, err
+	}
+	for _, a := range asked {
+		if a.Name == name {
+			return map[string]any{"name": name, "routed": false, "asked": a,
+				"says": fmt.Sprintf("no proxy serves %s; the proxy on %s answered \"no route\" for it, last at %s",
+					name, a.Proxy, a.AskedAt.UTC().Format(time.RFC3339))}, nil
+		}
+	}
+	return map[string]any{"name": name, "routed": false,
+		"says": fmt.Sprintf("no proxy serves %s, and none was asked for it this week", name)}, nil
+}
+
+func (f front) refusals(node string) (map[string]any, error) {
+	asked, err := read[routestate.Unrouted](f.unrouted, node)
+	if err != nil {
+		return nil, err
+	}
+	sort.Slice(asked, func(i, j int) bool { return asked[i].AskedAt.After(asked[j].AskedAt) })
+	if asked == nil {
+		asked = []routestate.Unrouted{}
+	}
+	return map[string]any{"unrouted": asked, "names": len(asked)}, nil
+}

modules/route-proxy/cmd/route-provider/tools_test.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"route-proxy/internal/routestate"
+)
+
+// held is a state as the bus holds it, read-only.
+type held map[string]any
+
+func (h held) Keys() ([]string, error) {
+	out := []string{}
+	for k := range h {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+func (h held) Get(key string) (*stdio.StateEntry, error) {
+	v, ok := h[key]
+	if !ok {
+		return nil, nil
+	}
+	raw, err := json.Marshal(v)
+	return &stdio.StateEntry{Key: key, Value: raw}, err
+}
+
+// The web front's tools answer from the bus, in one call each (issue 774): every site with its target, certificate
+// expiry and last answer; one site in full, or that it is not served and when it was asked for; and the names the
+// proxy answered "no route for".
+func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) {
+	now := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
+	expires := now.Add(30 * 24 * time.Hour)
+	routes := held{
+		"anchor.grafana.example": routestate.Route{Name: "grafana.example", Proxy: "anchor", Reach: routestate.ReachPublic,
+			Targets:     []routestate.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}},
+			Certificate: routestate.Certificate{Authority: routestate.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: routestate.Issued},
+			LastAnswer:  &routestate.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}},
+		"anchor.drive.example": routestate.Route{Name: "drive.example", Proxy: "anchor", Reach: routestate.ReachPublic,
+			Targets:     []routestate.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}},
+			Certificate: routestate.Certificate{Authority: routestate.ReachPublic},
+			LastAnswer:  &routestate.Answer{Status: 200, At: now}},
+	}
+	unrouted := held{
+		"anchor.old.example":  routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3},
+		"anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1},
+	}
+	tools := map[string]stdio.Tool{}
+	for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() {
+		tools[tool.Name] = tool
+	}
+
+	for _, row := range []struct {
+		tool string
+		args map[string]any
+		want []string // what the answer says, in its JSON
+	}{
+		{RoutesTool, nil, []string{`"served":2`, `"unreachable":1`,
+			`"name":"grafana.example"`, `"reachable":"no"`, `"expires_in_days":30`, `/ → grafana@home-server at http://10.0.0.2:3000`,
+			`"name":"drive.example"`, `"reachable":"yes"`, `public: none seen yet`}},
+		{RoutesTool, map[string]any{"node": "home-server"}, []string{`"served":0`}},
+		{RouteTool, map[string]any{"name": "Grafana.Example"}, []string{`"routed":true`, `"reachable":"no"`,
+			`connection refused`, `"issuer":"Let's Encrypt R11"`}},
+		{RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`,
+			`answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}},
+		{RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}},
+		{UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`}},
+	} {
+		t.Run(row.tool, func(t *testing.T) {
+			tool, ok := tools[row.tool]
+			if !ok {
+				t.Fatalf("%s is not served", row.tool)
+			}
+			if !strings.Contains(tool.Description, "route-proxy") {
+				t.Errorf("%s's description does not name the web front, so a search does not find it", row.tool)
+			}
+			answer, err := tool.Run(row.args)
+			if err != nil {
+				t.Fatal(err)
+			}
+			raw, _ := json.Marshal(answer)
+			for _, w := range row.want {
+				if !strings.Contains(string(raw), w) {
+					t.Errorf("%s(%v) does not say %s:\n%s", row.tool, row.args, w, raw)
+				}
+			}
+		})
+	}
+}

modules/route-proxy/cmd/route-proxy/authority_test.go

@@ -19,14 +19,15 @@ const (
 // path back: nothing is re-registered, no order reaches the CA, and issuance stops with nothing
 // saying why. Naming the cache after the authority means the account is only ever found where it is
 // still valid — the new root lands in a directory with no account in it, and autocert registers.
-func TestANewCARootMeansANewAccountCache(t *testing.T) {
+func TestTheAccountCacheIsNamedAfterItsAuthority(t *testing.T) {
+	t.Run("a new CA root means a new account cache", func(t *testing.T) {
 		const directory = "https://anchor.internal/acme/acme/directory"
 		before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
 		after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot))
 		if before == after {
 			t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before)
 		}
-}
+	})
 
 	// And the SAME authority keeps the account it registered, restart after restart.
 	//
@@ -34,7 +35,7 @@ func TestANewCARootMeansANewAccountCache(t *testing.T) {
 	// certificates that did not persist would be re-ordered on every restart, which works silently until
 	// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the
 	// mesh writes that file, and a trailing newline coming or going must not throw away an account.
-func TestTheSameAuthorityKeepsItsAccount(t *testing.T) {
+	t.Run("the same authority keeps its account", func(t *testing.T) {
 		const directory = "https://anchor.internal/acme/acme/directory"
 		first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
 		again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n"))
@@ -42,25 +43,25 @@ func TestTheSameAuthorityKeepsItsAccount(t *testing.T) {
 			t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s",
 				first, again)
 		}
-}
+	})
 
 	// Staging and production are different authorities, and were sharing one account.
 	//
 	// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against
 	// staging reused the staging account, because the cache had no idea they were different.
-func TestStagingAndProductionDoNotShareAnAccount(t *testing.T) {
+	t.Run("staging and production do not share an account", func(t *testing.T) {
 		staging := forThisAuthority("/acme", stagingDirectory, nil)
 		production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil)
 		if staging == production {
 			t.Errorf("two issuers share one account: %s", staging)
 		}
-}
+	})
 
 	// It stays inside the directory the mesh gave it, and is a plain name.
 	//
 	// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or
 	// that carried a separator out of the PEM — would put an account somewhere nothing persists.
-func TestTheAccountCacheStaysWhereTheMeshPutIt(t *testing.T) {
+	t.Run("the account cache stays where the mesh put it", func(t *testing.T) {
 		const cache = "/var/lib/route-proxy/acme"
 		got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot))
 		if !strings.HasPrefix(got, cache+"/") {
@@ -70,4 +71,5 @@ func TestTheAccountCacheStaysWhereTheMeshPutIt(t *testing.T) {
 		if name != filepath.Base(got) || strings.ContainsAny(name, "/.") {
 			t.Errorf("the account cache is not a plain name: %q", name)
 		}
+	})
 }

modules/route-proxy/cmd/route-proxy/bus.go

@@ -10,10 +10,23 @@ import (
 
 	"github.com/nats-io/nats.go"
 
+	"git.novox.be/novox/mesh-sdk/go/announce"
 	"git.novox.be/novox/mesh-sdk/go/membership"
 	"git.novox.be/novox/mesh-sdk/go/process"
+
+	"route-proxy/internal/routestate"
 )
 
+// keptStates is every state route-proxy keeps, as both its halves write them: the routes and the refusals this
+// proxy writes, readable by others, and the bound entries the provider helper writes.
+func keptStates() []announce.State {
+	return []announce.State{
+		{Name: routestate.BoundState},
+		{Name: routestate.RoutesState, Readable: true},
+		{Name: routestate.UnroutedState, Readable: true},
+	}
+}
+
 // What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
 //
 // **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
@@ -44,6 +57,14 @@ func followOnce(path string, held *table, fromBus *atomic.Bool) error {
 	if err != nil {
 		return fmt.Errorf("connecting to the bus at %s: %w", membership.BareAddress(account.URL), err)
 	}
+	// What this proxy keeps, said on the bus (novox/hq ADR 0495; issue 774): the names it serves and the names it
+	// refused, readable by any module granted a read of them; the provider helper's entries, read by the machines'
+	// clients through their own grant. Said before the first write, so no write of an unannounced state is refused.
+	if err := conn.Keeps(keptStates()...); err != nil {
+		log.Printf("cannot say on the bus what this proxy keeps, so it keeps nothing there: %v", err)
+	} else {
+		held.ledger.attach(account.Node, conn.State(routestate.RoutesState), conn.State(routestate.UnroutedState))
+	}
 	subject := membership.Subject(account.Node, account.Module)
 	err = conn.FollowMembership(func(issued membership.Membership) {
 		if took := applyMembership(issued, held); took && !fromBus.Swap(true) {

modules/route-proxy/cmd/route-proxy/ledger.go

@@ -0,0 +1,478 @@
+package main
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"log"
+	"net"
+	"net/http"
+	"reflect"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/acme/autocert"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"route-proxy/internal/routestate"
+)
+
+// What this proxy says on the bus about the names it serves and the names it refuses (issue 774).
+//
+// **On change, never on a timer.** A route arriving or leaving, a certificate issued or failing, an upstream answer
+// that differs from the last one written, a name asked for that is not served: each marks its key, and one writer
+// puts the marked keys. An answer that is the same as the last one written is written again only once it is older
+// than refreshAfter, so a site answering 200 all day costs one write a minute, not one per request.
+//
+// Until the bus is reached nothing is written; what happened meanwhile is held and written when it is.
+
+// refreshAfter is how old a written answer, refusal or certificate failure may be before the same one is written
+// again with its new time.
+const refreshAfter = time.Minute
+
+// mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes.
+const mostAsked = 4096
+
+// kvStore is one of the module's states, as process.State is one.
+type kvStore interface {
+	Put(key string, value any) (uint64, error)
+	Delete(key string) error
+	Keys() ([]string, error)
+	Get(key string) (*stdio.StateEntry, error)
+}
+
+// ledger is what this proxy keeps of what it serves and refuses, and writes to the bus.
+type ledger struct {
+	mu  sync.Mutex
+	now func() time.Time
+
+	node             string
+	routes, unrouted kvStore
+
+	served   map[string]*routestate.Route
+	previous map[string]routestate.Route // what the bus held for this machine when it was reached
+	asked    map[string]*routestate.Unrouted
+
+	answerWritten map[string]time.Time
+	askedWritten  map[string]time.Time
+
+	dirtyRoutes map[string]bool // a name to put when served, to delete when not
+	dirtyAsked  map[string]bool
+
+	kick chan struct{}
+}
+
+func newLedger() *ledger {
+	return &ledger{
+		now:           time.Now,
+		served:        map[string]*routestate.Route{},
+		previous:      map[string]routestate.Route{},
+		asked:         map[string]*routestate.Unrouted{},
+		answerWritten: map[string]time.Time{},
+		askedWritten:  map[string]time.Time{},
+		dirtyRoutes:   map[string]bool{},
+		dirtyAsked:    map[string]bool{},
+		kick:          make(chan struct{}, 1),
+	}
+}
+
+// loop writes whatever was marked, each time something was.
+func (l *ledger) loop() {
+	for range l.kick {
+		l.flush()
+	}
+}
+
+func (l *ledger) wake() {
+	select {
+	case l.kick <- struct{}{}:
+	default:
+	}
+}
+
+// attach is the bus reached, as node: what the bus held for this machine is read back, so a restart keeps a name's
+// certificate and last answer, and everything held is written.
+func (l *ledger) attach(node string, routes, unrouted kvStore) {
+	if l == nil {
+		return
+	}
+	before := map[string]routestate.Route{}
+	if keys, err := routes.Keys(); err != nil {
+		log.Printf("cannot read back what this proxy said it serves: %v", err)
+	} else {
+		for _, key := range keys {
+			at, name := routestate.Split(key)
+			if at != node {
+				continue
+			}
+			entry, err := routes.Get(key)
+			if err != nil || entry == nil {
+				continue
+			}
+			var r routestate.Route
+			if json.Unmarshal(entry.Value, &r) == nil {
+				before[name] = r
+			}
+		}
+	}
+	l.mu.Lock()
+	l.node, l.routes, l.unrouted = node, routes, unrouted
+	for name, r := range before {
+		l.previous[name] = r
+		l.dirtyRoutes[name] = true // put again when served, deleted when no longer
+		if now := l.served[name]; now != nil {
+			keepWhatWasKnown(now, r)
+		}
+	}
+	for name := range l.served {
+		l.dirtyRoutes[name] = true
+	}
+	for name := range l.asked {
+		l.dirtyAsked[name] = true
+	}
+	l.mu.Unlock()
+	l.wake()
+}
+
+// keepWhatWasKnown carries a certificate and a last answer read back from the bus onto a route that knows none yet.
+func keepWhatWasKnown(r *routestate.Route, before routestate.Route) {
+	if r.Certificate.Outcome == "" && r.Certificate.NotAfter == nil && before.Certificate.Authority == r.Certificate.Authority {
+		r.Certificate = before.Certificate
+	}
+	if r.LastAnswer == nil {
+		r.LastAnswer = before.LastAnswer
+	}
+}
+
+// serving is the table this proxy now serves: a name new or changed is written, a name gone is deleted, and a name
+// that was asked for unserved and now is served leaves the unrouted state.
+func (l *ledger) serving(routes map[string][]rule, public map[string]bool) {
+	if l == nil {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for name, rules := range routes {
+		reach := routestate.ReachInternal
+		if public[name] {
+			reach = routestate.ReachPublic
+		}
+		targets := targetsOf(rules)
+		r := l.served[name]
+		if r == nil {
+			r = &routestate.Route{Name: name}
+			if before, ok := l.previous[name]; ok {
+				r.Certificate.Authority = reach
+				keepWhatWasKnown(r, before)
+			}
+			l.served[name] = r
+			l.dirtyRoutes[name] = true
+		}
+		if r.Reach != reach || !reflect.DeepEqual(r.Targets, targets) {
+			r.Reach, r.Targets = reach, targets
+			l.dirtyRoutes[name] = true
+		}
+		if r.Certificate.Authority != reach {
+			r.Certificate = routestate.Certificate{Authority: reach}
+			l.dirtyRoutes[name] = true
+		}
+		if _, was := l.asked[name]; was {
+			delete(l.asked, name)
+			l.dirtyAsked[name] = true
+		}
+	}
+	for name := range l.served {
+		if _, still := routes[name]; !still {
+			delete(l.served, name)
+			delete(l.answerWritten, name)
+			l.dirtyRoutes[name] = true
+		}
+	}
+	l.wake()
+}
+
+func targetsOf(rules []rule) []routestate.Target {
+	out := make([]routestate.Target, 0, len(rules))
+	for _, r := range rules {
+		out = append(out, routestate.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target,
+			Deny: r.policy.deny, Redirect: r.policy.redirectTo, Auth: len(r.policy.users) > 0 || r.policy.sealed != "",
+			Sealed: r.policy.sealed})
+	}
+	return out
+}
+
+// answered is an upstream answer to a request for name: its status, or the error that stood in for one.
+func (l *ledger) answered(name, path string, status int, failed error) {
+	if l == nil {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	r := l.served[name]
+	if r == nil {
+		return
+	}
+	now := l.now()
+	why := ""
+	if failed != nil {
+		why = failed.Error()
+	}
+	before := r.LastAnswer
+	r.LastAnswer = &routestate.Answer{Status: status, At: now, Error: why, Path: path}
+	if before != nil && before.Status == status && before.Error == why && now.Sub(l.answerWritten[name]) < refreshAfter {
+		return
+	}
+	l.answerWritten[name] = now
+	l.dirtyRoutes[name] = true
+	l.wake()
+}
+
+// refused is a name this proxy answered "no route for": in a request, or in a handshake's server name.
+func (l *ledger) refused(name, from string, internalOnly, handshake bool) {
+	if l == nil || !routestate.Keyable(name) {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if _, isServed := l.served[name]; isServed && !internalOnly {
+		return
+	}
+	now := l.now()
+	a := l.asked[name]
+	if a == nil {
+		if len(l.asked) >= mostAsked {
+			l.forgetOldestAsked()
+		}
+		a = &routestate.Unrouted{Name: name}
+		l.asked[name] = a
+	}
+	if host, _, err := net.SplitHostPort(from); err == nil {
+		from = host
+	}
+	a.AskedAt, a.From, a.InternalOnly, a.Handshake = now, from, internalOnly, handshake
+	a.Times++
+	if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter {
+		return
+	}
+	l.askedWritten[name] = now
+	l.dirtyAsked[name] = true
+	l.wake()
+}
+
+// forgetOldestAsked drops the name asked longest ago from memory; the bus keeps it until its time to live ends.
+func (l *ledger) forgetOldestAsked() {
+	oldest := ""
+	for name, a := range l.asked {
+		if oldest == "" || a.AskedAt.Before(l.asked[oldest].AskedAt) {
+			oldest = name
+		}
+	}
+	delete(l.asked, oldest)
+	delete(l.askedWritten, oldest)
+	delete(l.dirtyAsked, oldest)
+}
+
+// certified is a certificate an authority issued for name, as autocert stored it.
+func (l *ledger) certified(name string, leaf *x509.Certificate) {
+	if l == nil {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	r := l.served[name]
+	if r == nil {
+		return
+	}
+	now := l.now()
+	notAfter := leaf.NotAfter.UTC()
+	r.Certificate.Issuer, r.Certificate.NotAfter = issuerOf(leaf), &notAfter
+	r.Certificate.RenewedAt, r.Certificate.At = &now, &now
+	r.Certificate.Outcome, r.Certificate.Error = routestate.Issued, ""
+	l.dirtyRoutes[name] = true
+	l.wake()
+}
+
+// handedOut is the certificate a handshake for name was answered with: written when it is not the one recorded.
+func (l *ledger) handedOut(name string, leaf *x509.Certificate) {
+	if l == nil || leaf == nil {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	r := l.served[name]
+	if r == nil {
+		return
+	}
+	notAfter := leaf.NotAfter.UTC()
+	if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != routestate.Failed {
+		return
+	}
+	now := l.now()
+	r.Certificate.Issuer, r.Certificate.NotAfter, r.Certificate.At = issuerOf(leaf), &notAfter, &now
+	if r.Certificate.Outcome != routestate.Issued || r.Certificate.Error != "" {
+		r.Certificate.Outcome, r.Certificate.Error = routestate.Cached, ""
+	}
+	l.dirtyRoutes[name] = true
+	l.wake()
+}
+
+// certificateFailed is a handshake for name that found no certificate to answer with, and why.
+func (l *ledger) certificateFailed(name string, failed error) {
+	if l == nil {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	r := l.served[name]
+	if r == nil {
+		return
+	}
+	now := l.now()
+	same := r.Certificate.Outcome == routestate.Failed && r.Certificate.Error == failed.Error()
+	if same && r.Certificate.At != nil && now.Sub(*r.Certificate.At) < refreshAfter {
+		return
+	}
+	r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = routestate.Failed, failed.Error(), &now
+	l.dirtyRoutes[name] = true
+	l.wake()
+}
+
+func issuerOf(leaf *x509.Certificate) string {
+	if cn := leaf.Issuer.CommonName; cn != "" {
+		if len(leaf.Issuer.Organization) > 0 {
+			return leaf.Issuer.Organization[0] + " " + cn
+		}
+		return cn
+	}
+	return leaf.Issuer.String()
+}
+
+// flush writes every marked key. A write that fails stays marked and is tried again at the next change.
+func (l *ledger) flush() {
+	type put struct {
+		store kvStore
+		key   string
+		value any // nil deletes
+	}
+	l.mu.Lock()
+	if l.routes == nil {
+		l.mu.Unlock()
+		return
+	}
+	var puts []put
+	for name := range l.dirtyRoutes {
+		p := put{store: l.routes, key: routestate.Key(l.node, name)}
+		if r := l.served[name]; r != nil {
+			c := *r
+			c.Proxy = l.node
+			p.value = c
+		}
+		puts = append(puts, p)
+	}
+	for name := range l.dirtyAsked {
+		p := put{store: l.unrouted, key: routestate.Key(l.node, name)}
+		if a := l.asked[name]; a != nil {
+			c := *a
+			c.Proxy = l.node
+			p.value = c
+		}
+		puts = append(puts, p)
+	}
+	l.dirtyRoutes, l.dirtyAsked = map[string]bool{}, map[string]bool{}
+	l.mu.Unlock()
+
+	for _, p := range puts {
+		var err error
+		if p.value == nil {
+			err = p.store.Delete(p.key)
+		} else {
+			_, err = p.store.Put(p.key, p.value)
+		}
+		if err != nil {
+			log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err)
+			_, name := routestate.Split(p.key)
+			l.mu.Lock()
+			if p.store == l.routes {
+				l.dirtyRoutes[name] = true
+			} else {
+				l.dirtyAsked[name] = true
+			}
+			l.mu.Unlock()
+		}
+	}
+}
+
+// recordingCache is an authority's certificate cache that tells the ledger of every certificate stored in it: the
+// one place autocert says it was issued one.
+type recordingCache struct {
+	autocert.Cache
+	held *table
+}
+
+func (c recordingCache) Put(ctx context.Context, key string, data []byte) error {
+	if err := c.Cache.Put(ctx, key, data); err != nil {
+		return err
+	}
+	name := strings.TrimSuffix(key, "+rsa")
+	if strings.Contains(name, "+") || !c.held.routed(name) {
+		return nil // the account key, a challenge token
+	}
+	if leaf := leafOf(data); leaf != nil {
+		c.held.ledger.certified(bareHost(name), leaf)
+	}
+	return nil
+}
+
+// leafOf is the first certificate in what autocert stores: the private key, then the chain, in PEM.
+func leafOf(data []byte) *x509.Certificate {
+	for {
+		var block *pem.Block
+		block, data = pem.Decode(data)
+		if block == nil {
+			return nil
+		}
+		if block.Type == "CERTIFICATE" {
+			leaf, err := x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return nil
+			}
+			return leaf
+		}
+	}
+}
+
+// answerRecorder carries a proxied request's answer through and remembers its status and the proxy's error.
+type answerRecorder struct {
+	http.ResponseWriter
+	status int
+	err    error
+}
+
+func (a *answerRecorder) WriteHeader(status int) {
+	if a.status == 0 {
+		a.status = status
+	}
+	a.ResponseWriter.WriteHeader(status)
+}
+
+func (a *answerRecorder) Write(b []byte) (int, error) {
+	if a.status == 0 {
+		a.status = http.StatusOK
+	}
+	return a.ResponseWriter.Write(b)
+}
+
+// Unwrap lets http.ResponseController reach the connection beneath: a flush, an upgrade's hijack.
+func (a *answerRecorder) Unwrap() http.ResponseWriter { return a.ResponseWriter }
+
+// proxyFailed is the reverse proxy's answer when upstream gave none: the error kept for the ledger, and a 502.
+func proxyFailed(w http.ResponseWriter, r *http.Request, err error) {
+	if a, ok := w.(*answerRecorder); ok {
+		a.err = err
+	}
+	log.Printf("http: proxy error for %s%s: %v", r.Host, r.URL.Path, err)
+	w.WriteHeader(http.StatusBadGateway)
+}

modules/route-proxy/cmd/route-proxy/ledger_test.go

@@ -0,0 +1,245 @@
+package main
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"testing"
+	"time"
+
+	"golang.org/x/crypto/acme/autocert"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"route-proxy/internal/routestate"
+)
+
+// fakeState is a state on the bus, in memory, counting what was written to it.
+type fakeState struct {
+	values map[string]json.RawMessage
+	puts   int
+}
+
+func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} }
+
+func (f *fakeState) Put(key string, value any) (uint64, error) {
+	raw, err := json.Marshal(value)
+	if err != nil {
+		return 0, err
+	}
+	f.values[key] = raw
+	f.puts++
+	return uint64(f.puts), nil
+}
+func (f *fakeState) Delete(key string) error { delete(f.values, key); return nil }
+func (f *fakeState) Keys() ([]string, error) {
+	out := []string{}
+	for k := range f.values {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+func (f *fakeState) Get(key string) (*stdio.StateEntry, error) {
+	v, ok := f.values[key]
+	if !ok {
+		return nil, nil
+	}
+	return &stdio.StateEntry{Key: key, Value: v}, nil
+}
+
+func (f *fakeState) route(t *testing.T, key string) *routestate.Route {
+	t.Helper()
+	raw, ok := f.values[key]
+	if !ok {
+		return nil
+	}
+	var r routestate.Route
+	if err := json.Unmarshal(raw, &r); err != nil {
+		t.Fatal(err)
+	}
+	return &r
+}
+
+// issuedPEM is what autocert stores for a name: the key, then the certificate, signed by an issuer named here.
+func issuedPEM(t *testing.T, name string, notAfter time.Time) []byte {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), DNSNames: []string{name},
+		Subject: pkix.Name{CommonName: name}, Issuer: pkix.Name{CommonName: "R11", Organization: []string{"Let's Encrypt"}},
+		NotBefore: notAfter.Add(-90 * 24 * time.Hour), NotAfter: notAfter}
+	issuer := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: template.Issuer, NotBefore: template.NotBefore,
+		NotAfter: notAfter, IsCA: true, BasicConstraintsValid: true}
+	der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyDER, _ := x509.MarshalECPrivateKey(key)
+	out := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
+}
+
+// What the proxy tells the bus (issue 774): each served name with where it goes, its certificate and the last
+// upstream answer; each name refused; and only on change. Without the ledger nothing reaches the bus, and the
+// operator's question "is grafana reachable" is answered with curl.
+func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) {
+	notAfter := time.Date(2027, 1, 28, 12, 0, 0, 0, time.UTC)
+	gone := func() string { // an address nothing listens on: the upstream that does not answer
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer l.Close()
+		return l.Addr().String()
+	}()
+	_, port, _ := net.SplitHostPort(gone)
+	contributions := []contribution{
+		{From: "grafana", Node: "anchor", At: "127.0.0.1", Values: map[string]any{"name": "grafana.example", "port": mustFloat(t, port)}},
+		{From: "drive", Node: "home-server", At: "10.0.0.2", Values: map[string]any{"internal-name": "drive.internal", "port": 8080.0}},
+	}
+
+	for _, row := range []struct {
+		name  string
+		act   func(t *testing.T, held *table)
+		check func(t *testing.T, routes, unrouted *fakeState)
+	}{
+		{"a served name is written with the module and node it forwards to",
+			func(t *testing.T, held *table) {},
+			func(t *testing.T, routes, _ *fakeState) {
+				r := routes.route(t, "anchor.grafana.example")
+				if r == nil || r.Proxy != "anchor" || r.Reach != routestate.ReachPublic || len(r.Targets) != 1 ||
+					r.Targets[0].Module != "grafana" || r.Targets[0].Node != "anchor" || r.Targets[0].Target != "http://127.0.0.1:"+port {
+					t.Fatalf("grafana's route is not on the bus as served: %+v", r)
+				}
+				if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != routestate.ReachInternal ||
+					d.Certificate.Authority != routestate.ReachInternal {
+					t.Fatalf("an internal-only name is not on the bus as internal: %+v", d)
+				}
+			}},
+		{"a withdrawn route is deleted",
+			func(t *testing.T, held *table) {
+				routes, public := routesOf(contributions[1:])
+				held.set(routes, public)
+			},
+			func(t *testing.T, routes, _ *fakeState) {
+				if routes.route(t, "anchor.grafana.example") != nil {
+					t.Fatal("a route no longer served is still on the bus")
+				}
+			}},
+		{"an upstream that does not answer is written as a 502 with why, once a minute",
+			func(t *testing.T, held *table) {
+				for range 3 {
+					req := httptest.NewRequest(http.MethodGet, "http://grafana.example/login", nil)
+					handler(held).ServeHTTP(httptest.NewRecorder(), req)
+				}
+			},
+			func(t *testing.T, routes, _ *fakeState) {
+				r := routes.route(t, "anchor.grafana.example")
+				if r == nil || r.LastAnswer == nil || r.LastAnswer.Status != http.StatusBadGateway ||
+					r.LastAnswer.Error == "" || r.LastAnswer.Path != "/login" {
+					t.Fatalf("the failed answer is not on the bus: %+v", r)
+				}
+				if routes.puts != 3 { // the two routes, then the first answer; the same answer again is not written
+					t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts)
+				}
+			}},
+		{"a name not served is written as asked, and leaves when it is served",
+			func(t *testing.T, held *table) {
+				req := httptest.NewRequest(http.MethodGet, "http://grafana.other.example/", nil)
+				req.RemoteAddr = "192.0.2.7:51000"
+				handler(held).ServeHTTP(httptest.NewRecorder(), req)
+				held.ledger.flush()
+				var a routestate.Unrouted
+				raw := held.ledger.unrouted.(*fakeState).values["anchor.grafana.other.example"]
+				if err := json.Unmarshal(raw, &a); err != nil || a.From != "192.0.2.7" || a.Times != 1 || a.AskedAt.IsZero() {
+					t.Fatalf("the refused name is not on the bus as asked: %s", raw)
+				}
+				routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor",
+					Values: map[string]any{"name": "grafana.other.example", "port": 3000.0}}))
+				held.set(routes, public)
+			},
+			func(t *testing.T, _, unrouted *fakeState) {
+				if len(unrouted.values) != 0 {
+					t.Fatalf("a name now served is still written as unrouted: %v", unrouted.values)
+				}
+			}},
+		{"a handshake for a name not routed is written as asked",
+			func(t *testing.T, held *table) {
+				pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+					return nil, errors.New("no public route for \"forgotten.example\" in this mesh, so no certificate is asked for")
+				}, nil)
+				_, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.example"})
+			},
+			func(t *testing.T, _, unrouted *fakeState) {
+				var a routestate.Unrouted
+				if err := json.Unmarshal(unrouted.values["anchor.forgotten.example"], &a); err != nil || !a.Handshake {
+					t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values)
+				}
+			}},
+		{"a certificate an authority issued is written with its issuer, expiry and renewal",
+			func(t *testing.T, held *table) {
+				cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held}
+				if err := cache.Put(context.Background(), "grafana.example", issuedPEM(t, "grafana.example", notAfter)); err != nil {
+					t.Fatal(err)
+				}
+				_ = cache.Put(context.Background(), "acme_account+key", []byte("not a certificate"))
+			},
+			func(t *testing.T, routes, _ *fakeState) {
+				c := routes.route(t, "anchor.grafana.example").Certificate
+				if c.Outcome != routestate.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil ||
+					!c.NotAfter.Equal(notAfter) || c.RenewedAt == nil {
+					t.Fatalf("the issued certificate is not on the bus: %+v", c)
+				}
+			}},
+		{"a certificate that cannot be had is written with why",
+			func(t *testing.T, held *table) {
+				pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+					return nil, errors.New("acme: urn:ietf:params:acme:error:rateLimited")
+				}, nil)
+				_, _ = pick(&tls.ClientHelloInfo{ServerName: "grafana.example"})
+			},
+			func(t *testing.T, routes, _ *fakeState) {
+				c := routes.route(t, "anchor.grafana.example").Certificate
+				if c.Outcome != routestate.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" {
+					t.Fatalf("the failed certificate is not on the bus: %+v", c)
+				}
+			}},
+	} {
+		t.Run(row.name, func(t *testing.T) {
+			held := newTable()
+			held.ledger = newLedger()
+			routes, unrouted := newFakeState(), newFakeState()
+			held.ledger.attach("anchor", routes, unrouted)
+			table, public := routesOf(contributions)
+			held.set(table, public)
+			held.ledger.flush()
+			row.act(t, held)
+			held.ledger.flush()
+			row.check(t, routes, unrouted)
+		})
+	}
+}
+
+func mustFloat(t *testing.T, port string) float64 {
+	t.Helper()
+	var n float64
+	if err := json.Unmarshal([]byte(port), &n); err != nil {
+		t.Fatal(err)
+	}
+	return n
+}

modules/route-proxy/cmd/route-proxy/main.go

@@ -179,6 +179,8 @@ type rule struct {
 	// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
 	// a proxy's own default refuses them long before the workload is reached.
 	maxRequestBody int64
+	// from and node are the consumer that contributed the rule and its machine, which the bus is told (issue 774).
+	from, node string
 }
 
 // table is what the proxy is currently serving, replaced whole whenever the file changes.
@@ -202,6 +204,8 @@ type table struct {
 	// machine's address on the private network, as the mesh issued it in this proxy's membership
 	// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
 	inside sources
+	// ledger is what this proxy says on the bus about what it serves and refuses (issue 774); nil says nothing.
+	ledger *ledger
 }
 
 // sources is who may be served an internal name: the private network's addresses as the mesh
@@ -289,6 +293,7 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) {
 	t.to = made
 	t.public = public
 	t.mu.Unlock()
+	t.ledger.serving(made, public)
 }
 
 // inOrder puts the rules for one host into the order they are matched in, and does so totally.
@@ -437,6 +442,8 @@ func run() error {
 	}
 
 	held := newTable()
+	held.ledger = newLedger()
+	go held.ledger.loop()
 	// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
 	// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
 	// it an internal name is served to this machine and to nobody else — refused, never opened.
@@ -487,7 +494,7 @@ func run() error {
 			"to persist, or every restart orders them again")
 	}
 	publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
-		onlyWhatTheMeshSaid(held))
+		onlyWhatTheMeshSaid(held), held)
 	if err != nil {
 		return err
 	}
@@ -499,7 +506,7 @@ func run() error {
 	var internalManager *autocert.Manager
 	if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
 		internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
-			onlyInternalNamesTheMeshSaid(held))
+			onlyInternalNamesTheMeshSaid(held), held)
 		if err != nil {
 			return fmt.Errorf("internal certificate authority: %w", err)
 		}
@@ -619,7 +626,7 @@ func (p *probedResponse) replayTo(w http.ResponseWriter) {
 // one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
 // with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
 // issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
-func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
+func newManager(cache, directory, bundle string, policy autocert.HostPolicy, held *table) (*autocert.Manager, error) {
 	client := &acme.Client{DirectoryURL: directory}
 	var root []byte
 	if bundle != "" {
@@ -651,7 +658,7 @@ func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*a
 	// different names because their directories differ.
 	mine := forThisAuthority(cache, directory, root)
 	return &autocert.Manager{
-		Cache:      autocert.DirCache(mine),
+		Cache:      recordingCache{Cache: autocert.DirCache(mine), held: held},
 		Prompt:     autocert.AcceptTOS,
 		HostPolicy: policy,
 		Client:     client,
@@ -700,9 +707,10 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
 	if internalManager != nil {
 		fromInternal = internalManager.TLSConfig().GetCertificate
 	}
-	return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
 		if held.eligibleForInternalACME(hello.ServerName) {
 			if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
+				held.ledger.refused(bareHost(hello.ServerName), hello.Conn.RemoteAddr().String(), true, true)
 				return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
 					hello.ServerName)
 			}
@@ -712,6 +720,32 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
 		}
 		return fromPublic(hello)
 	}
+	// What the bus is told of a handshake (issue 774): a routed name's certificate, or why it has none; a name not
+	// routed here, which a browser asking for a site the mesh forgot to route meets before any request is made.
+	return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		cert, err := pick(hello)
+		name := bareHost(hello.ServerName)
+		switch {
+		case name == "":
+		case !held.routed(name):
+			from := ""
+			if hello.Conn != nil {
+				from = hello.Conn.RemoteAddr().String()
+			}
+			held.ledger.refused(name, from, false, true)
+		case err != nil:
+			if !strings.HasPrefix(err.Error(), "no public route for") {
+				held.ledger.certificateFailed(name, err)
+			}
+		case cert != nil:
+			leaf := cert.Leaf
+			if leaf == nil && len(cert.Certificate) > 0 {
+				leaf, _ = x509.ParseCertificate(cert.Certificate[0])
+			}
+			held.ledger.handedOut(name, leaf)
+		}
+		return cert, err
+	}
 }
 
 // newTable is an empty routing table.
@@ -737,6 +771,9 @@ func handler(held *table) http.Handler {
 			// this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the
 			// jail's filter expects it.
 			log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
+			if hidden || !held.routed(r.Host) {
+				held.ledger.refused(bareHost(r.Host), r.RemoteAddr, hidden, false)
+			}
 			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 			w.WriteHeader(http.StatusNotFound)
 			if !hidden && held.routed(r.Host) {
@@ -786,7 +823,14 @@ func handler(held *table) http.Handler {
 			r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
 		}
 
-		matched.to.ServeHTTP(w, r)
+		// The upstream's answer is what the bus is told of this name (issue 774): its status, or why there was none.
+		answer := &answerRecorder{ResponseWriter: w}
+		matched.to.ServeHTTP(answer, r)
+		status := answer.status
+		if status == 0 && answer.err == nil && r.Header.Get("Upgrade") != "" {
+			status = http.StatusSwitchingProtocols
+		}
+		held.ledger.answered(bareHost(r.Host), r.URL.Path, status, answer.err)
 	})
 }
 
@@ -955,6 +999,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
 			}
 			made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
 		}
+		made.from, made.node = c.From, c.Node
 
 		if name != "" {
 			host := strings.ToLower(name)
@@ -1072,5 +1117,5 @@ func towards(where *url.URL) *httputil.ReverseProxy {
 		pr.SetURL(where)
 		pr.Out.Host = pr.In.Host
 		pr.SetXForwarded()
-	}}
+	}, ErrorHandler: proxyFailed}
 }

modules/route-proxy/cmd/route-proxy/routes_test.go

@@ -121,19 +121,6 @@ func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
 	}
 }
 
-// A route with neither name has nothing to be served under, and is still skipped.
-func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
-	routes, public, err := routesFrom(write(t, `{"given":[
-	  {"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":"  ","port":8080}}
-	]}`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(routes) != 0 || len(public) != 0 {
-		t.Errorf("a route that named nothing was served: %v %v", routes, public)
-	}
-}
-
 // A route with no internal-name composed gets no second host — the ordinary case, unchanged.
 func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
 	routes, _, err := routesFrom(write(t, `{"given":[
@@ -161,21 +148,6 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
 	}
 }
 
-// Skipped rather than served wrongly. A route with no port would proxy to :0.
-func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
-	routes, _, err := routesFrom(write(t, `{"given":[
-	  {"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
-	  {"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
-	  {"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}
-	]}`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(routes) != 1 || targetOf(routes, "fine.example") == "" {
-		t.Fatalf("an unusable contribution was served: %v", routes)
-	}
-}
-
 // A route may name a target reached over https, for a backend that terminates its own TLS — the
 // shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise.
 func TestARouteMayTargetHttps(t *testing.T) {
@@ -194,19 +166,6 @@ func TestARouteMayTargetHttps(t *testing.T) {
 	}
 }
 
-// A scheme that is neither http nor https is refused rather than guessed at.
-func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) {
-	routes, _, err := routesFrom(write(t, `{"given":[
-	  {"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}
-	]}`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(routes) != 0 {
-		t.Fatalf("a route with an unusable scheme was served: %v", routes)
-	}
-}
-
 // End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still
 // reached when the route declared `insecure`, and the response comes back through unmodified.
 func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) {
@@ -455,23 +414,6 @@ func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
 	}
 }
 
-// A limit that is not a whole positive number of bytes takes the route with it. Serving it without
-// the limit would carry exactly what the module said not to carry, and report success doing it.
-func TestARouteWithAnUnusableLimitIsSkipped(t *testing.T) {
-	for _, asked := range []string{`"lots"`, `-1`, `0`, `1.5`} {
-		routes, _, err := routesFrom(write(t, `{"given":[
-		  {"from":"registry","node":"anchor","at":"anchor.internal",
-		   "values":{"name":"images.example","port":5000,"max-request-body":`+asked+`}}
-		]}`))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if len(routes["images.example"]) != 0 {
-			t.Errorf("a route asking for a max-request-body of %s was served anyway: %v", asked, routes)
-		}
-	}
-}
-
 // A request larger than the route carries is refused by the proxy, with the limit named, and the
 // workload never sees it. A request within it is proxied normally.
 func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
@@ -532,3 +474,32 @@ func post(url, host, body string) (*http.Response, error) {
 	asked.Header.Set("Content-Type", "application/octet-stream")
 	return http.DefaultClient.Do(asked)
 }
+
+// A contribution that cannot be served is skipped rather than served wrongly: one that names nothing to be served
+// under; one with no port, which would proxy to :0; a scheme that is neither http nor https, refused rather than
+// guessed at; and a body limit that is not a whole positive number of bytes, since serving it without the limit would
+// carry exactly what the module said not to carry, and report success doing it.
+func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) {
+	for _, row := range []struct{ why, given, served string }{
+		{"neither name", `{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":"  ","port":8080}}`, ""},
+		{"no port", `{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
+		  {"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
+		  {"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}`, "fine.example"},
+		{"an unknown scheme", `{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}`, ""},
+		{"a limit in words", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":"lots"}}`, ""},
+		{"a negative limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":-1}}`, ""},
+		{"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""},
+		{"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""},
+	} {
+		routes, public, err := routesFrom(write(t, `{"given":[`+row.given+`]}`))
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch {
+		case row.served == "" && (len(routes) != 0 || len(public) != 0):
+			t.Errorf("%s: an unusable contribution was served: %v %v", row.why, routes, public)
+		case row.served != "" && (len(routes) != 1 || targetOf(routes, row.served) == ""):
+			t.Errorf("%s: an unusable contribution was served beside the usable one: %v", row.why, routes)
+		}
+	}
+}

modules/route-proxy/internal/routestate/routestate.go

@@ -0,0 +1,122 @@
+// Package routestate is what the web front keeps on the bus about the names it serves, and about the names it was asked
+// for and does not serve (issue 774): the shape both halves of route-proxy agree on. The proxy writes it, on change,
+// in its states `routes` and `unrouted`, announced readable; the module's tools read it from there, so the question
+// "is this site reachable, and is its certificate good" is answered from the bus rather than with curl.
+//
+// A key is `<node>.<name>`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds
+// no dot, so the first dot splits them.
+package routestate
+
+import (
+	"regexp"
+	"strings"
+	"time"
+)
+
+// The states route-proxy keeps, by their local names (the buckets are route-proxy_<name>).
+const (
+	// RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got.
+	RoutesState = "routes"
+	// UnroutedState holds one key per name the proxy answered "no route for" with, and when it was last asked.
+	UnroutedState = "unrouted"
+	// BoundState is the provider helper's bucket of consumer entries (mesh-sdk go/provider).
+	BoundState = "bound"
+)
+
+// The words a route's reach and a certificate's authority are said in.
+const (
+	ReachPublic   = "public"
+	ReachInternal = "internal"
+)
+
+// The words a certificate's outcome is said in.
+const (
+	// Issued: the authority issued it while this proxy ran, at RenewedAt.
+	Issued = "issued"
+	// Cached: this proxy served it from what it kept, and saw no issuance of it since it started.
+	Cached = "cached"
+	// Failed: the last attempt to have one failed, Error says why.
+	Failed = "failed"
+)
+
+// Target is one rule a name is served by: what a request matching it reaches, or what it is answered with.
+type Target struct {
+	// Path is the path prefix the rule is scoped to; empty is every path.
+	Path string `json:"path,omitempty"`
+	// Module and Node are the consumer that contributed the route and the machine it runs on.
+	Module string `json:"module,omitempty"`
+	Node   string `json:"node,omitempty"`
+	// Target is where the request is forwarded, scheme://address:port; empty for a rule that answers itself.
+	Target   string `json:"target,omitempty"`
+	Deny     bool   `json:"deny,omitempty"`
+	Redirect string `json:"redirect,omitempty"`
+	// Auth says the rule asks for credentials; Sealed says why it cannot check them, and so refuses everything.
+	Auth   bool   `json:"auth,omitempty"`
+	Sealed string `json:"sealed,omitempty"`
+}
+
+// Certificate is what this proxy knows of a name's certificate.
+type Certificate struct {
+	// Authority is which issuer certifies the name: public (the ACME directory the module names) or internal (the
+	// mesh's own).
+	Authority string     `json:"authority"`
+	Issuer    string     `json:"issuer,omitempty"`
+	NotAfter  *time.Time `json:"not_after,omitempty"`
+	// RenewedAt is when an authority last issued it to this proxy.
+	RenewedAt *time.Time `json:"renewed_at,omitempty"`
+	// Outcome is issued, cached or failed; empty until a handshake or an issuance said anything.
+	Outcome string     `json:"outcome,omitempty"`
+	Error   string     `json:"error,omitempty"`
+	At      *time.Time `json:"at,omitempty"`
+}
+
+// Answer is the last answer the proxy got from upstream for a name: the status it carried back, or the error that
+// stood in for one (a 502).
+type Answer struct {
+	Status int       `json:"status"`
+	At     time.Time `json:"at"`
+	Error  string    `json:"error,omitempty"`
+	Path   string    `json:"path,omitempty"`
+}
+
+// Route is one served name, the value of a key of RoutesState.
+type Route struct {
+	Name string `json:"name"`
+	// Proxy is the machine whose proxy serves it.
+	Proxy string `json:"proxy"`
+	// Reach is public (a route's own name) or internal (only its internal-name, served inside the private network).
+	Reach       string      `json:"reach"`
+	Targets     []Target    `json:"targets"`
+	Certificate Certificate `json:"certificate"`
+	LastAnswer  *Answer     `json:"last_answer,omitempty"`
+}
+
+// Unrouted is one name the proxy answered "no route for", the value of a key of UnroutedState.
+type Unrouted struct {
+	Name  string `json:"name"`
+	Proxy string `json:"proxy"`
+	// AskedAt is when the name was last asked for; From is who asked last.
+	AskedAt time.Time `json:"asked_at"`
+	From    string    `json:"from,omitempty"`
+	// Times is how often it was asked since the proxy last started.
+	Times int `json:"times"`
+	// InternalOnly says the name is served, but only inside the private network, and was asked from outside.
+	InternalOnly bool `json:"internal_only,omitempty"`
+	// Handshake says it was asked in a TLS handshake (the name a client sent), not in an HTTP request.
+	Handshake bool `json:"handshake,omitempty"`
+}
+
+var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`)
+
+// Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request
+// is whatever a client sent; one no key could hold is no site anybody runs.
+func Keyable(name string) bool { return keyable.MatchString(name) && !strings.Contains(name, "..") }
+
+// Key is the key of name served or asked at node's proxy.
+func Key(node, name string) string { return node + "." + name }
+
+// Split is a key's node and name.
+func Split(key string) (node, name string) {
+	node, name, _ = strings.Cut(key, ".")
+	return node, name
+}

modules/route-proxy/module.json

@@ -244,8 +244,29 @@
     }
   ],
   "state": [
-    "bound"
+    "bound",
+    "routes",
+    {
+      "name": "unrouted",
+      "ttl-seconds": 604800
+    }
   ],
+  "tools": [
+    "route_proxy_routes",
+    "route_proxy_route",
+    "route_proxy_unrouted"
+  ],
+  "replaces": {
+    "route_proxy_routes": [
+      "curl -I"
+    ],
+    "route_proxy_route": [
+      "openssl s_client -connect"
+    ],
+    "route_proxy_unrouted": [
+      "docker logs route-proxy"
+    ]
+  },
   "reads": [
     "mesh-controller.assignments"
   ]