Compare commits

..
Author SHA1 Message Date
mesh-tools 4c9be8efc8 Merge pull request 'The web front's route helper reads the seat's plans, so a route-only module's route is provided (issue 774)' (#486) from fix/route-proxy-reads-plans into main 2026-10-30 23:43:15 +00:00
jschoubben 62c05c681c Start route-proxy's provider helper only once Serve holds the runtime's channel, so its first watch is never refused and the helper never stops until a restart (issue 774, review)
mesh/delivery merged: source.main moved to novox/mesh-controller@4c9be8efc83a
mesh/check checking 62c05c681cfb on 4c9be8efc83a
2026-10-31 00:32:26 +01:00
jschoubben bfb728730f Answer route-proxy's routes from the seat mesh-provisioner's plans and keys, not the controller's assignments, so a route the seat keys for a module not handed is marked provided and its new builds are no longer withheld (issue 774) 2026-10-31 00:27:11 +01:00
mesh-tools 159db37618 Merge pull request 'The web front keeps what it serves, its certificates and what it refuses on the bus, with tools to read them (issue 774)' (#485) from feat/route-proxy-tools into main 2026-10-30 22:11:52 +00:00
jschoubben b7c3868980 Keep a scan out of route-proxy's unrouted state: only names under a served domain, a few a minute, no visitor address (issue 774)
mesh/delivery merged: source.main moved to novox/mesh-controller@159db37618c2
mesh/check being carried to b7c3868980f3 on 159db37618c2 | from a4426190b0c7
Review of ddae6729: a scanner's random Host or SNI names each became a key, bounded only by the
bucket's lifetime and size, and every failed write was retried at each change and logged each time.

A refused name is now kept by name only under the registrable domain of a name the proxy serves, and
at most five new names a minute; every other refusal is counted in memory and written once a minute
under <node>._other (last minute, since start), which route_proxy_unrouted shows. A refused write of
that state is dropped and logged once until a write succeeds; trimming memory to 4096 names moved
from the request to the writer. The visitor's address is no longer kept: the journal has it.

Quick: drop unrouted keys from strangers by a fixed allow-list. Thorough: bound by the proxy's own
domains, by rate, and by never retrying. Chose thorough: the scan is constant on a public machine.
2026-10-30 23:03:00 +01:00
jschoubben ddae6729c1 Give route-proxy its state on the bus and three tools, so a site's route, certificate and last answer are read from the mesh instead of with curl (issue 774)
The web front kept nothing on the bus: whether grafana or drive was routed, where to, with which
certificate, and what it last answered could only be learned with curl or the container's log.

The proxy now keeps `routes` and `unrouted` (announced readable, keyed <node>.<name>), written on
change by one writer: route set changes, certificates stored by autocert's cache or failing at a
handshake, upstream answers that differ from the last written (or are a minute old), and names
refused in a request or a handshake. The provider helper's bundle serves route_proxy_routes,
route_proxy_route and route_proxy_unrouted from those states.

Quick: log the routes at start and grep the journal. Thorough: the state on the bus and the tools
that read it, mesh-wide, in one call. Chose thorough: the operator's question is mesh-wide and
recurs, and the log says nothing about certificates or answers.

Tests: two table tests added (the ledger, the tools), both failing without the change; four
authority tests and four skipped-route tests merged into two, so the module has four fewer.
2026-10-30 22:58:04 +01:00
58 changed files with 2132 additions and 1011 deletions
+5 -22
View File
@@ -30,11 +30,6 @@ const packageManagerKey = "the package manager ("
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
//
// **The bound gates raising, never keeping** (sweep 2026-10-30, issue 774). A report says a failure again whenever its
// words change, with a new time: on novox the vault's refusal of every secret replaced a bus refusal at 16:56, and the
// condition cleared for five minutes while every module still failed. Once raised, it stays raised while any failure
// stands on that machine, judged by the open condition the keeper holds on the bus, not by a report's time.
func watchApplyFailing(f *signalFacts) []conditions.Observation {
heard := map[string]bool{}
for _, m := range f.machines {
@@ -45,7 +40,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
said := map[string]bool{}
for _, d := range f.doing {
if !d.Wrong() || d.Since == nil || !heard[d.Node] || (f.now.Sub(*d.Since) <= applyEvery && !f.applyFailingOpen[d.Node]) {
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
continue
}
out = append(out, applyFailing(d, f.now))
@@ -56,8 +51,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
// on a machine whose node-engine applied what it was sent.
failed := map[string][]inventory.ProvReport{}
for _, r := range f.provFailed {
if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] ||
(f.now.Sub(r.Since()) <= applyEvery && !f.applyFailingOpen[r.Node]) {
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
continue
}
failed[r.Node] = append(failed[r.Node], r)
@@ -68,28 +62,17 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
return out
}
// applyFailingOpen are the machines whose S21 condition is open among the open conditions.
func applyFailingOpen(open []conditions.Condition) map[string]bool {
out := map[string]bool{}
for _, c := range open {
if c.Kind == kindApplyFailing && c.Subject.Machine != "" {
out[c.Subject.Machine] = true
}
}
return out
}
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
// place.
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
var names, words []string
since := failed[0].Since()
since := failed[0].At
for _, r := range failed {
names = append(names, r.Module)
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
if r.Since().Before(since) {
since = r.Since()
if r.At.Before(since) {
since = r.At
}
}
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
+1 -1
View File
@@ -250,7 +250,7 @@ func prepareSend(ctx context.Context, open *stores, node string) ([]string, erro
if n.Stated && n.Contract >= compose.VaultBoxContract {
boxes, err := coreBoxesFor(ctx, inv, node, resolved.Plan.Modules, func(m catalogue.Manifest) bool {
return handing.Handing(m)
}, true, in.BusMinted)
}, true)
if err != nil {
return nil, err
}
+2 -2
View File
@@ -124,11 +124,11 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
return
}
if moved := builtByTheMesh(moved); len(moved) > 0 {
if len(moved) > 0 {
fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
node, strings.Join(moved, ", "))
}
if left := builtByTheMesh(left); len(left) > 0 {
if len(left) > 0 {
fmt.Printf("compositions: %s's %s are left out of the node's own: the controller knows no fingerprint of the "+
"new form for them, and the node's want or have names their own composition (issue 1945)\n",
node, strings.Join(left, ", "))
+4 -12
View File
@@ -69,15 +69,7 @@ type coreBoxes struct {
// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries,
// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the
// client, and one issued outside the mesh is never the vault's to seal here.
//
// **Nor a process's bus account nobody issued yet** (sweep 2026-10-30, issue 774). Asked with `get`, the vault holds
// none and makes a random value under the account's name: own/novox/nats/broker-nats-tools at 16:36:51. Until the
// controller issues the account and hands it to the vault, the machine has the controller's stand-in (prepareSend), as
// every module below the vault's contract does. (The controller's own broker-address and broker-management are no
// accounts: the bus's address, which broker.FromEnvironment reads, and the retired AMQP management login; no process
// declares them, so this leaves them asked as before.)
func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool, node string,
minted []string) map[string][]string {
func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string {
out := map[string][]string{}
for _, m := range modules {
if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) {
@@ -85,7 +77,7 @@ func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest)
}
var names []string
for name, own := range m.OwnSecrets.Plain() {
if own.IssuedBy != catalogue.IssuedOutside && processAccountWarning(m, node, name, minted) == "" {
if own.IssuedBy != catalogue.IssuedOutside {
names = append(names, name)
}
}
@@ -216,8 +208,8 @@ var coreVault = func(ctx context.Context) coreAsk {
// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send.
func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest,
handed func(catalogue.Manifest) bool, allocating bool, minted []string) (coreBoxes, error) {
core := coreModulesOf(modules, handed, node, minted)
handed func(catalogue.Manifest) bool, allocating bool) (coreBoxes, error) {
core := coreModulesOf(modules, handed)
if len(core) == 0 {
if allocating {
settleVaultSilent(ctx, node)
+1 -28
View File
@@ -124,35 +124,8 @@ func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) {
{Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}},
{Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}},
}
got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" }, "anchor", nil)
got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" })
if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) {
t.Fatalf("got %v, want %v", got, want)
}
}
// Sweep 2026-10-30 (issue 774): a bus process's account is asked of the vault only once it is issued; before, the vault
// would make a random value under the account's name.
func TestAnUnissuedProcessAccountIsNotAskedOfTheVault(t *testing.T) {
nats, err := catalogue.ParseManifest([]byte(`{"module":"nats","version":"1","emits":["x"],
"own-secrets":{"broker":{"path":"/a"},"broker-nats-tools":{"path":"/b","owner":"nats-tools"}},
"resources":[{"id":"account","type":"user","name":"nats-tools"},
{"id":"nats-tools","type":"process","run":["./nats-tools"],"user":"nats-tools","bus":{"emits":["x"]}}]}`))
if err != nil {
t.Fatal(err)
}
if len(nats.ProcessBuses()) == 0 {
t.Fatal("the fixture declares no bus process")
}
for name, c := range map[string]struct {
minted []string
want []string
}{
"not issued": {nil, []string{"broker"}},
"issued": {[]string{processUser("anchor", "nats", "nats-tools")}, []string{"broker", "broker-nats-tools"}},
} {
got := coreModulesOf([]catalogue.Manifest{nats}, nil, "anchor", c.minted)
if !reflect.DeepEqual(got["nats"], c.want) {
t.Errorf("%s: asks %v, want %v", name, got["nats"], c.want)
}
}
}
@@ -1,31 +0,0 @@
//go:build integration
package main
import (
"context"
"testing"
"github.com/nats-io/nats.go/jetstream"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/novox/mesh-controller/internal/broker"
)
// sayForgeMerge is the forge bridge saying a merge on its stream FORGE, which it makes when missing.
func sayForgeMerge(ctx context.Context, t *testing.T, js *broker.JetStream, repo string, number int, head, merge string) {
t.Helper()
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
if _, err := api.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: forgeStream,
Subjects: []string{process.ModuleStreamSubject(sdkdelivery.ForgeBridgeModule, "forge.>")}}); err != nil {
t.Fatal(err)
}
subject, body := forgeMerged(t, repo, number, head, merge)
if _, err := api.Publish(ctx, subject, body); err != nil {
t.Fatal(err)
}
}
-106
View File
@@ -1,106 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"github.com/nats-io/nats.go/jetstream"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
)
// fakeForge is the forge bridge's stream as forgeMergeAt reads it: messages by sequence, each on a subject.
type fakeForge struct {
subjects []string
data [][]byte
}
func (f fakeForge) last(_ context.Context, subject string) (uint64, []byte, error) {
for i := len(f.subjects) - 1; i >= 0; i-- {
if f.subjects[i] == subject {
return uint64(i + 1), f.data[i], nil
}
}
return 0, nil, jetstream.ErrMsgNotFound
}
func (f fakeForge) next(_ context.Context, subject string, from uint64) (uint64, []byte, error) {
for i := int(from) - 1; i < len(f.subjects); i++ {
if i >= 0 && f.subjects[i] == subject {
return uint64(i + 1), f.data[i], nil
}
}
return 0, nil, jetstream.ErrMsgNotFound
}
func forgeMerged(t *testing.T, repo string, number int, head, merge string) (string, []byte) {
t.Helper()
owner, name, _ := strings.Cut(repo, "/")
subject, err := forgeMergeSubject(owner, name)
if err != nil {
t.Fatal(err)
}
body, err := json.Marshal(sdkdelivery.Forge{V: 1, Kind: sdkdelivery.ForgeMerged, Repository: repo, Branch: "main",
Number: number, Head: head, Merge: merge, Root: "t0", Trees: map[string]string{".": "t1"}})
if err != nil {
t.Fatal(err)
}
return subject, body
}
// **A moved module's merge is read from the forge bridge's stream** (sweep 2026-10-30, issue 774): the controller keeps
// no copy of the forge's merges, so its registration finds the pull request and head of a merge commit on FORGE, the
// newest first and then every merge the stream keeps; a merge of another repository or commit is never taken for it.
func TestAMergeIsFoundOnTheForgeBridgesStream(t *testing.T) {
var f fakeForge
add := func(repo string, number int, head, merge string) {
s, b := forgeMerged(t, repo, number, head, merge)
f.subjects, f.data = append(f.subjects, s), append(f.data, b)
}
add("novox/mesh-host", 6, "h6", "c6c6c6c6c6")
add("novox/mesh-catalog", 9, "h9", "c7c7c7c7c7")
add("novox/mesh-host", 7, "h7", "c7c7c7c7c7")
f.subjects, f.data = append(f.subjects, f.subjects[0]), append(f.data, []byte("not a forge message"))
add("novox/mesh-host", 8, "h8", "c8c8c8c8c8")
for name, c := range map[string]struct {
repo, commit string
number int
found bool
}{
"the newest merge": {"novox/mesh-host", "c8c8c8c8c8", 8, true},
"an older merge, found by scanning": {"novox/mesh-host", "c6c6c6c6c6", 6, true},
"a short commit": {"novox/mesh-host", "c7c7c7c", 7, true},
"a commit of another repository": {"novox/mesh-catalog", "c6c6c6c6c6", 0, false},
"a repository with no merge": {"novox/hq", "c6c6c6c6c6", 0, false},
} {
owner, name2, _ := strings.Cut(c.repo, "/")
k, found, err := forgeMergeAt(t.Context(), f, owner, name2, c.commit)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if found != c.found || k.Number != c.number {
t.Errorf("%s: found %v #%d, want %v #%d", name, found, k.Number, c.found, c.number)
}
if found && (k.Owner != owner || k.Repo != name2 || k.HeadSHA == "") {
t.Errorf("%s: the merge reads as %+v", name, k.SourceMoved)
}
}
}
// Sweep 2026-10-30 (issue 774): a module the mesh provides itself is never named as lacking a fingerprint.
func TestAModuleTheMeshProvidesIsNotSaidToLackAFingerprint(t *testing.T) {
for _, c := range []struct {
in, want []string
}{
{[]string{"mesh-wireguard"}, nil},
{[]string{"claude-code", "mesh-wireguard", "power"}, []string{"claude-code", "power"}},
{nil, nil},
} {
if got := builtByTheMesh(c.in); strings.Join(got, ",") != strings.Join(c.want, ",") {
t.Errorf("%v: %v, want %v", c.in, got, c.want)
}
}
}
+1 -54
View File
@@ -21,13 +21,8 @@ func holdingOne(names ...string) func(broker.Consumer) (uint64, error) {
}
func deafObservations(now time.Time, held func(broker.Consumer) (uint64, error)) (string, int) {
return deafHeld(now, time.Time{}, held)
}
// deafHeld is deafObservations with the loop's last hold due at due.
func deafHeld(now, due time.Time, held func(broker.Consumer) (uint64, error)) (string, int) {
f := calm(now)
loop, err := readLoop(now.Add(-loopDeafAfter-time.Minute), due, loopConsumers(), held)
loop, err := readLoop(now.Add(-loopDeafAfter-time.Minute), loopConsumers(), held)
if err != nil {
panic(err)
}
@@ -70,51 +65,3 @@ func TestDeafLookReadsExactlyTheControllersOwnConsumers(t *testing.T) {
}
}
}
// Sweep 2026-10-30 (issue 774): a message the loop held back with a delay longer than the deaf bound is not a deaf
// controller until it is due and still not taken; the merges consumer's five-minute holds flapped the alarm this way.
func TestAMessageHeldBackIsNotDeafUntilItIsDue(t *testing.T) {
now := time.Now()
held := holdingOne("CONTROL/" + broker.ControllerName)
for name, c := range map[string]struct {
due time.Time
deaf int
}{
"nothing held": {time.Time{}, 1},
"held, due in three minutes": {now.Add(3 * time.Minute), 0},
"held, due a moment ago": {now.Add(-time.Second), 0},
"held, due past the deaf bound, unsent": {now.Add(-loopDeafAfter - time.Second), 1},
} {
if _, n := deafHeld(now, c.due, held); n != c.deaf {
t.Errorf("%s: %d deaf observation(s), want %d", name, n, c.deaf)
}
}
}
// Sweep 2026-10-30 (issue 774): the controller holds no consumer of the forge's merges any more, and removes the one
// it held at every assertion.
func TestTheMergesConsumerIsRetired(t *testing.T) {
for _, c := range broker.MeshConsumers() {
if c.Name == broker.MergesConsumerName {
t.Fatalf("the controller still holds %s on %s", c.Name, c.Stream)
}
}
r := &removed{}
if err := broker.AssertMeshConsumers(r); err != nil {
t.Fatal(err)
}
if !r.gone[broker.EventsStream+"/"+broker.MergesConsumerName] {
t.Fatalf("the retired merges consumer is not removed: removed %v", r.gone)
}
}
type removed struct{ gone map[string]bool }
func (*removed) EnsureConsumer(broker.Consumer) error { return nil }
func (r *removed) RemoveConsumer(stream, name string) error {
if r.gone == nil {
r.gone = map[string]bool{}
}
r.gone[stream+"/"+name] = true
return nil
}
+42 -98
View File
@@ -16,8 +16,6 @@ import (
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/kvlimit"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
@@ -181,12 +179,7 @@ func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, emitter st
if err != nil {
return fmt.Errorf("%w: the bucket of mesh-delivery's merges: %w", link.ErrTryAgain, err)
}
// Through kvlimit (sweep 2026-10-30, issue 774): a word too large for the bucket is refused before it is put, and
// is an answer, never "not now" — retried, it held the forge's merges for a day (10054 on de-spiegel's merge).
if _, err := kvlimit.Put(ctx, kv, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
if errors.Is(err, kvlimit.ErrTooLarge) {
return err
}
if _, err := kv.Put(ctx, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
}
if k.kept != nil {
@@ -350,113 +343,64 @@ func claimantsOf(entries []inventory.Entry, seat string) []string {
return out
}
// keptForgeMerge is the forge's word on the merge of a repository at a commit: the pull request and the head it merged,
// read from the forge bridge's own stream, FORGE, where mesh-delivery-forge-bridge says every merge as
// `forge.<owner>.<name>.merged` and keeps it a week (novox/hq ADR 0437). False when the stream holds no merge at that
// commit. A variable so a test can hand it one.
//
// **The controller keeps no copy of the forge's merges any more** (sweep 2026-10-30, issue 774). It once kept each in
// its bucket of merges off a consumer of its own on EVENTS that never gave up a message; one announcement over the
// bucket's value size held that consumer for a day and queued every merge behind it, and the deaf watchdog read the
// queue as a deaf controller. The forge bridge's stream is the record of the merges (research 072, 03-domains: the
// delivery parts own what a merge is); the controller reads it where it needs one, at a moved module's registration.
// forgeMergeKey is where the forge's announcement of a merge is kept, by the merge commit: the one thing a build of a merge
// names, from which the pull request and its head are found (novox/hq ADR 0420 §7).
func forgeMergeKey(owner, repo, commit string) string {
return fmt.Sprintf("merge/%s/%s/%s", owner, repo, commit)
}
// keepForgeMerge keeps the forge's announcement of a merge in the bucket of the merges, a week at most as that bucket
// keeps them. The controller plans nothing from a merge any more (ADR 0539 §17); this is all it keeps of one.
var keepForgeMerge = func(ctx context.Context, m link.SourceMoved) error {
serving := servingBus.Load()
if serving == nil {
return fmt.Errorf("%w: the bus is not held", link.ErrTryAgain)
}
api, err := jetstream.New(serving.Conn())
if err != nil {
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
}
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
if err != nil {
return fmt.Errorf("%w: the bucket of the merges: %w", link.ErrTryAgain, err)
}
body, err := json.Marshal(keptMerge{SourceMoved: m})
if err != nil {
return err
}
if _, err := kv.Put(ctx, forgeMergeKey(m.Owner, m.Repo, m.Commit), body); err != nil {
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
}
return nil
}
// keptForgeMerge is the forge's announcement of a merge, by its commit; false when none was kept. A variable so a test can
// hand it one.
var keptForgeMerge = func(ctx context.Context, owner, repo, commit string) (keptMerge, bool, error) {
serving := servingBus.Load()
if serving == nil {
return keptMerge{}, false, errors.New("only the serving controller reads the forge's merges")
return keptMerge{}, false, errors.New("only the serving controller keeps what the forge announced")
}
api, err := jetstream.New(serving.Conn())
if err != nil {
return keptMerge{}, false, err
}
stream, err := api.Stream(ctx, forgeStream)
if err != nil {
return keptMerge{}, false, fmt.Errorf("the forge bridge's stream %s cannot be read: %w", forgeStream, err)
}
return forgeMergeAt(ctx, streamMerges{stream}, owner, repo, commit)
}
// forgeStream is the forge bridge's stream as the bus names it (its declared name, forge, in capitals).
var forgeStream = catalogue.StreamBusName(sdkdelivery.ForgeStream)
// forgeMergeSubject is the subject of the forge bridge's merged message for a repository.
func forgeMergeSubject(owner, repo string) (string, error) {
event, err := sdkdelivery.ForgeEvent(owner+"/"+repo, sdkdelivery.ForgeMerged)
if err != nil {
return "", err
}
return process.ModuleStreamSubject(sdkdelivery.ForgeBridgeModule, event), nil
}
// mergeMessages is what forgeMergeAt reads of a stream: the last message on a subject, and the first at or after a
// sequence. Each answers jetstream.ErrMsgNotFound when there is none.
type mergeMessages interface {
last(ctx context.Context, subject string) (uint64, []byte, error)
next(ctx context.Context, subject string, from uint64) (uint64, []byte, error)
}
type streamMerges struct{ s jetstream.Stream }
func (m streamMerges) last(ctx context.Context, subject string) (uint64, []byte, error) {
msg, err := m.s.GetLastMsgForSubject(ctx, subject)
if err != nil {
return 0, nil, err
}
return msg.Sequence, msg.Data, nil
}
func (m streamMerges) next(ctx context.Context, subject string, from uint64) (uint64, []byte, error) {
msg, err := m.s.GetMsg(ctx, from, jetstream.WithGetMsgSubject(subject))
if err != nil {
return 0, nil, err
}
return msg.Sequence, msg.Data, nil
}
// forgeMergeAt finds the merged message of owner/repo at commit: the newest first, which is the merge a take-in
// follows nearly always, then every one the stream keeps, oldest first. A message that does not read as a forge
// message is passed over.
func forgeMergeAt(ctx context.Context, msgs mergeMessages, owner, repo, commit string) (keptMerge, bool, error) {
subject, err := forgeMergeSubject(owner, repo)
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
if err != nil {
return keptMerge{}, false, err
}
// Read for the four fields a registration needs, not checked whole (sdkdelivery.DecodeForge): a merged message a
// newer bridge writes, or one the SDK's check refuses for a part the controller never reads, still names its pull
// request, head and commit.
match := func(data []byte) (keptMerge, bool) {
var f sdkdelivery.Forge
if json.Unmarshal(data, &f) != nil || f.Kind != sdkdelivery.ForgeMerged || f.Number <= 0 || f.Head == "" ||
!sameCommitID(f.Merge, commit) {
return keptMerge{}, false
}
return keptMerge{SourceMoved: link.SourceMoved{Owner: owner, Repo: repo, Base: f.Branch, Number: f.Number,
HeadSHA: f.Head, Commit: f.Merge}}, true
}
lastSeq, data, err := msgs.last(ctx, subject)
if errors.Is(err, jetstream.ErrMsgNotFound) {
e, err := kv.Get(ctx, forgeMergeKey(owner, repo, commit))
if errors.Is(err, jetstream.ErrKeyNotFound) {
return keptMerge{}, false, nil
}
if err != nil {
return keptMerge{}, false, err
}
if k, ok := match(data); ok {
return k, true, nil
var k keptMerge
if err := json.Unmarshal(e.Value(), &k); err != nil {
return keptMerge{}, false, err
}
for from := uint64(1); from < lastSeq; {
seq, data, err := msgs.next(ctx, subject, from)
if errors.Is(err, jetstream.ErrMsgNotFound) || (err == nil && seq >= lastSeq) {
return keptMerge{}, false, nil
}
if err != nil {
return keptMerge{}, false, err
}
if k, ok := match(data); ok {
return k, true, nil
}
from = seq + 1
}
return keptMerge{}, false, nil
return k, true, nil
}
// keptMerge is a merge as the record keeps it: the forge's announcement.
+10 -2
View File
@@ -99,9 +99,17 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
return err == nil && kept
})
// The forge bridge's merge of that pull request on its stream, and the build it asked for.
// The forge's merge of that pull request, as the controller heard it, and the build it asked for.
const commit = "c0ffee0123456789"
sayForgeMerge(ctx, t, js, "novox/mesh-host", 7, a.merge.HeadSHA, commit)
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
var forgeMerge link.SourceMoved
if err := json.Unmarshal(event, &forgeMerge); err != nil {
t.Fatal(err)
}
if err := keepForgeMerge(ctx, forgeMerge); err != nil {
t.Fatal(err)
}
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
Path: "modules/sudo", Ref: "main", Commit: commit, For: "merge"}); err != nil {
+10 -2
View File
@@ -100,9 +100,17 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB
return err == nil && kept
})
// The forge bridge's merge of that pull request on its stream, and the build it asked for.
// The forge's merge of that pull request, as the controller heard it, and the build it asked for.
const commit = "c0ffee0123456789"
sayForgeMerge(ctx, t, js, "novox/mesh-delivery", 7, a.merge.HeadSHA, commit)
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
var forgeMerge link.SourceMoved
if err := json.Unmarshal(event, &forgeMerge); err != nil {
t.Fatal(err)
}
if err := keepForgeMerge(ctx, forgeMerge); err != nil {
t.Fatal(err)
}
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-delivery", Seat: "git",
Path: "modules/mesh-delivery-view", Ref: "main", Commit: commit, For: "merge"}); err != nil {
@@ -11,6 +11,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// ADR 0501 §6 (issue 1851, A5): the live case of 2026-10-07 (novox/hq issue 300), on keys. A merge of the catalogue adds
@@ -32,6 +33,12 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T)
watchNewModules(watching, open)
const commit = "3da80a4b00000000"
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
// The indexer writes the module's trunk value; the builder builds it, then its key is done.
fp := fpOf('7')
manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp)
-42
View File
@@ -7,7 +7,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
@@ -73,44 +72,3 @@ func TestAFailedPlacingRaisesApplyFailing(t *testing.T) {
t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got)
}
}
// Sweep 2026-10-30 (issue 774): a failure said again in new words carries a new time, and the condition it raised stays
// raised while it stands; only a machine with no open condition waits out the bound, and a cleared failure clears it.
func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) {
now := time.Date(2026, 10, 30, 16, 57, 0, 0, time.UTC)
since := now.Add(-time.Minute)
for name, c := range map[string]struct {
open bool
prov string
doing bool
raised bool
since time.Duration // the client's failing-since, before now; zero when it says none
}{
"a fresh placing failure, nothing open": {false, inventory.ProvFailed, false, false, 0},
"a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true, 0},
"placing applied again, condition open": {true, inventory.ProvApplied, false, false, 0},
"a fresh failed apply, condition open": {true, "", true, true, 0},
"a fresh failed apply, nothing open": {false, "", true, false, 0},
"new words of a failure standing ten minutes, nothing open": {false, inventory.ProvFailed, false, true, 10 * time.Minute},
} {
f := calm(now)
if c.open {
f.applyFailingOpen = applyFailingOpen([]conditions.Condition{{Kind: kindApplyFailing,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}}})
}
if c.prov != "" {
f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov,
Reason: "the vault refused it", At: now.Add(-time.Minute)}}
if c.since > 0 {
f.provFailed[0].FailingSince = now.Add(-c.since)
}
}
if c.doing {
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since,
Since: &since}}
}
if got := watchApplyFailing(f); (len(got) == 1) != c.raised || len(got) > 1 {
t.Errorf("%s: %d observation(s), raised want %v", name, len(got), c.raised)
}
}
}
-54
View File
@@ -1,54 +0,0 @@
package main
import (
"context"
"fmt"
"regexp"
"github.com/novox/mesh-controller/internal/conditions"
)
// **A condition whose check no longer exists is cleared** (sweep 2026-10-30, issue 774). A signals row or a self-check
// probe raises and clears its conditions by reconciling what it sees; one removed from the controller reconciles
// nothing again, so what it raised stayed open for good and read as current: S23's failed plans (the plans went with
// ADR 0539) and D14's stalled deliveries, from 2026-10-20 on. Each watchdogs tick clears every open condition whose
// source is a row's or a probe's name that neither the signals table nor the probe registry holds any more, saying so.
// A source of another form (an event, a module's health, a verb) is never judged here: its own word clears it.
// checkSource is the form of a row's (S1) or a probe's (D4, DW, D-root) name.
var checkSource = regexp.MustCompile(`^(S[0-9]+|D[0-9]+|D[A-Z]+|D-[a-z]+)$`)
// liveChecks are the sources this controller still reconciles: its signals rows and its self-check's probes.
func liveChecks() map[string]bool {
out := map[string]bool{}
for _, r := range signalsTable {
out[r.Row] = true
}
for _, p := range probeRegistry {
out[p.ID] = true
}
return out
}
// retiredCheckConditions are the open conditions raised by a row or a probe the controller no longer has.
func retiredCheckConditions(open []conditions.Condition, live map[string]bool) []conditions.Condition {
var out []conditions.Condition
for _, c := range open {
if checkSource.MatchString(c.Source) && !live[c.Source] {
out = append(out, c)
}
}
return out
}
// clearRetiredChecks clears each of them, saying why.
func clearRetiredChecks(ctx context.Context, k *conditions.Keeper, retired []conditions.Condition) []string {
var problems []string
for _, c := range retired {
why := fmt.Sprintf("%s, which raised it, is no longer a check of the controller", c.Source)
if _, err := k.ClearSaying(ctx, c.Key, why, "No longer checked: the check that raised this was removed"); err != nil {
problems = append(problems, c.Key+": "+err.Error())
}
}
return problems
}
@@ -1,31 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/conditions"
)
// Sweep 2026-10-30 (issue 774): an open condition raised by a row or a probe the controller no longer has is cleared;
// one raised by a check it still has, or by a source of another form, is left to its own word.
func TestAConditionWhoseCheckIsGoneIsCleared(t *testing.T) {
live := liveChecks()
for source, gone := range map[string]bool{
"S23": true,
"D14": true,
"S21": false,
"S9": false,
"D-root": false,
"DM": false,
"DB": false,
"D4": false,
"health": false,
"provisioner.failing": false,
"done keys": false,
} {
got := retiredCheckConditions([]conditions.Condition{{Key: "k." + source, Source: source}}, live)
if (len(got) == 1) != gone {
t.Errorf("%s: judged retired %v, want %v", source, len(got) == 1, gone)
}
}
}
+1 -5
View File
@@ -284,17 +284,13 @@ func watchLoop(f *signalFacts) []conditions.Observation {
if since.IsZero() {
since = f.started
}
// A message the loop held back waits by the loop's own choice until it is due: deafness is counted from then.
if f.loop.due.After(since) {
since = f.loop.due
}
if f.now.Sub(since) <= loopDeafAfter {
return nil
}
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
"message(s): reports and events are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
}
+1 -14
View File
@@ -160,7 +160,7 @@ func switchList(ctx context.Context, inv *inventory.Inventory, keys switchReads)
var out []switchLine
for _, e := range entries {
m := e.Manifest.Module
if e.Provided || e.Source.Repository == "" || sdkdelivery.ProvidedByMesh(m) {
if e.Provided || e.Source.Repository == "" {
continue
}
trunk, err := keys.ModuleAt(ctx, m, "")
@@ -202,16 +202,3 @@ func heldArtifacts(ctx context.Context, inv *inventory.Inventory, module, builtF
// empty-on-error: not an error; the mesh holds no build of that commit on record, which the line says
return nil, nil
}
// builtByTheMesh are the modules of a list that a build of the mesh's makes: without the ones the mesh provides itself
// (sdkdelivery.ProvidedByMesh, the private network), which no fingerprint ever names, so a line saying one has none
// says nothing (sweep 2026-10-30, issue 774: "mesh-wireguard ... no fingerprint of the new form" on every composition).
func builtByTheMesh(modules []string) []string {
var out []string
for _, m := range modules {
if !sdkdelivery.ProvidedByMesh(m) {
out = append(out, m)
}
}
return out
}
+9
View File
@@ -93,6 +93,15 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
return out, nil
}
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
// A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and
// the builder builds them (ADR 0539 §5, §17). The controller keeps the announcement, for the build of a merge that
// moves a module between repositories to be shown the operator's Approve (ADR 0420 §7).
return keepForgeMerge(ctx, m)
}
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
var actingOnMerges sync.Mutex
+4 -14
View File
@@ -58,11 +58,6 @@ type signalFacts struct {
// provFailed is every module whose files a machine's provisioner client last said it failed to place (S21,
// novox/hq ADR 0362 §3): that machine's failed apply.
provFailed []inventory.ProvReport
// applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever
// the newest report's time (sweep 2026-10-30).
applyFailingOpen map[string]bool
// retiredChecks are the open conditions raised by a row or a probe the controller no longer has (sweep 2026-10-30).
retiredChecks []conditions.Condition
loop loopFacts
loopErr error
@@ -152,9 +147,7 @@ type machineFacts struct {
func (m machineFacts) asleep() bool { return m.power.Away() }
type loopFacts struct {
took time.Time
// due is when a message the loop held back is handed over again (link.Loop.Due): the loop is not deaf before it.
due time.Time
took time.Time
pending uint64
where string
}
@@ -259,7 +252,6 @@ func (w *watchdogs) see(running context.Context, f *signalFacts) {
problems = append(problems, "S8: "+err.Error())
}
}
problems = append(problems, clearRetiredChecks(running, w.keeper, f.retiredChecks)...)
if err := w.keeper.EndSilences(running); err != nil {
problems = append(problems, err.Error())
}
@@ -323,8 +315,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
} else {
f.standings = providerStandings(open)
f.providerWords = providerConditions(open)
f.applyFailingOpen = applyFailingOpen(open)
f.retiredChecks = retiredCheckConditions(open, liveChecks())
}
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
@@ -468,7 +458,7 @@ func (w *watchdogs) gatherLoop() (loopFacts, error) {
if w.js == nil {
return loopFacts{took: link.Loop.Last()}, errors.New("this controller is not on the bus")
}
return readLoop(link.Loop.Last(), link.Loop.Due(), loopConsumers(), func(c broker.Consumer) (uint64, error) {
return readLoop(link.Loop.Last(), loopConsumers(), func(c broker.Consumer) (uint64, error) {
info, err := w.js.Context().ConsumerInfo(c.Stream, c.Name)
if err != nil {
return 0, err
@@ -478,8 +468,8 @@ func (w *watchdogs) gatherLoop() (loopFacts, error) {
}
// readLoop sums what the given consumers hold, as held reads each.
func readLoop(took, due time.Time, consumers []broker.Consumer, held func(broker.Consumer) (uint64, error)) (loopFacts, error) {
f := loopFacts{took: took, due: due}
func readLoop(took time.Time, consumers []broker.Consumer, held func(broker.Consumer) (uint64, error)) (loopFacts, error) {
f := loopFacts{took: took}
var where []string
for _, c := range consumers {
n, err := held(c)
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+17
View File
@@ -41,6 +41,23 @@ func ControllerAnnouncement(node string) announce.States {
return s
}
// ReadsAnnounced splits a reader's `reads` (`<owner>.<name>`) by what each owner announced: granted when the owner
// announced the state readable, or announced nothing yet (the transition); refused when the owner announced and did
// not announce that state readable.
func ReadsAnnounced(reads []string, announced map[string]announce.States) (granted, refused []string) {
for _, r := range reads {
owner, name, ok := strings.Cut(r, ".")
s, said := announced[owner]
switch {
case !ok || !said || s.Readable(name):
granted = append(granted, r)
default:
refused = append(refused, r)
}
}
return granted, refused
}
// NotAnnounced are the owners some module reads the state of and that have announced nothing yet, sorted: read from the
// readers' `reads` as before, until they announce.
func NotAnnounced(r Records) []string {
+17
View File
@@ -7,6 +7,23 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce"
)
// D3 (ADR 0495 clause 3, ADR 0513 §3; issue 1790): a reader's `reads` is granted where the owner announced the state
// readable, refused where it announced and did not, and granted as before where the owner announced nothing yet.
func TestAReadIsGrantedFromItsOwnersAnnouncement(t *testing.T) {
announced := map[string]announce.States{
"owner": {Module: "owner", States: []announce.State{{Name: "open", Readable: true}, {Name: "closed"}}},
}
granted, refused := ReadsAnnounced([]string{"owner.open", "owner.closed", "owner.unsaid", "quiet.any"}, announced)
if !slices.Equal(granted, []string{"owner.open", "quiet.any"}) || !slices.Equal(refused, []string{"owner.closed", "owner.unsaid"}) {
t.Fatalf("granted %v, refused %v", granted, refused)
}
r := Records{Announced: announced, Assigned: map[string][]Declared{"one": {{Module: "reader", Reads: granted,
ReadsRefused: refused}}}}
if got := NotAnnounced(r); !slices.Equal(got, []string{"quiet"}) {
t.Fatalf("not yet announced: %v", got)
}
}
// D1 (ADR 0495 clause 2): the controller announces every bucket of its own, and readable exactly the record mirrors (the
// declared packages the pacman module reads among them, ADR 0507 §5), the chain's buckets, the checks running and the
// plans building.
-39
View File
@@ -1,39 +0,0 @@
package broker
import "testing"
// Sweep 2026-10-30 (issue 774), the guard of mesh-sdk #136 in the controller's copy: where a placement names the
// node-engine's manifest, it adds no user, no process and nothing the tool runner carries; a module beside it does.
func TestTheNodeEnginesManifestComposesNoPlacedUser(t *testing.T) {
engine := Declared{Module: EngineModule, Serves: []string{"health"}, State: []Bucket{{Name: "health"}},
Processes: []ProcessPrincipal{{Process: "apply"}}}
beside := Declared{Module: "power", Serves: []string{"power_draw"}}
for name, c := range map[string]struct {
assigned []Declared
module bool
}{
"with the runner": {[]Declared{{Module: RuntimeModule, NoAccount: true}, engine, beside}, false},
"without the runner": {[]Declared{engine, beside}, true},
} {
users, err := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": c.assigned}})
if err != nil {
t.Fatalf("%s: %v", name, err)
}
besideSeen := false
for _, u := range users {
if u.Module == EngineModule && (u.Kind == KindModule || u.Kind == KindProcess) {
t.Errorf("%s: the node-engine's manifest composed a %v user", name, u.Kind)
}
besideSeen = besideSeen || u.Module == beside.Module
for _, carried := range u.Carries {
if carried.Module == EngineModule {
t.Errorf("%s: the tool runner carries the node-engine's manifest", name)
}
besideSeen = besideSeen || carried.Module == beside.Module
}
}
if !besideSeen {
t.Errorf("%s: the module beside it is composed nowhere", name)
}
}
}
+22 -34
View File
@@ -339,36 +339,13 @@ func Overlaps() []string {
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
const ControllerName = "controller"
// MergesConsumerName was the controller's durable consumer of the forge's merged event on EVENTS (novox/hq ADR 0407
// decision 3, issue 973), with no max-deliver and one message at a time. **Retired** (sweep 2026-10-30, issue 774):
// the forge bridge's stream FORGE is the record of the merges, and the controller reads it where it needs a merge. On
// 2026-10-29 one announcement too large for the controller's bucket of merges was held on it for over a day, with
// 163 merges queued behind it, and the deaf watchdog read the queue as a deaf controller. RetiredConsumers removes it.
// MergesConsumerName is the controller's durable consumer of the forge's merged event on EVENTS, apart from the
// events consumer (novox/hq ADR 0407 decision 3, issue 973). A merge is never dropped, so this consumer has no
// max-deliver, which would send a merge held five times to DEAD_LETTERS to wait for a person, and no healer
// resets it, which would drop the merges it holds. A consumer of its own because both of those are properties of
// a consumer, not of a subject.
const MergesConsumerName = "controller-merges"
// RetiredConsumers are durable consumers the controller once held and holds no more: each assertion removes them,
// so a retired consumer cannot keep a queue nobody takes from (research 072, 03-domains: the controller shrinks).
var RetiredConsumers = []Consumer{{Name: MergesConsumerName, Stream: EventsStream}}
// Remover is the part of a JetStream connection that removes a retired consumer.
type Remover interface {
RemoveConsumer(stream, name string) error
}
// removeRetired removes every retired consumer, when the connection can.
func removeRetired(e Ensurer) error {
r, ok := e.(Remover)
if !ok {
return nil
}
for _, c := range RetiredConsumers {
if err := r.RemoveConsumer(c.Stream, c.Name); err != nil {
return fmt.Errorf("removing the retired consumer %s on %s: %w", c.Name, c.Stream, err)
}
}
return nil
}
// ControllerSeat is the role the control plane holds, and ControllerStates are the facts it states
// under it (novox/hq ADR 0134).
//
@@ -536,6 +513,23 @@ func ControllerConsumers() []Consumer {
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
"max-deliver, because the store window's bound is the controller's own",
},
// The forge's merges (novox/hq ADR 0407 decision 3, issue 973): never dropped, so no max-deliver and no
// reset; a merge the controller cannot record yet is held with a growing delay until it can. **Asserted before
// the events consumer**, which stops filtering merges: the order of this list is the order of assertion.
{
Name: MergesConsumerName,
Stream: "EVENTS",
// Within what the controller may subscribe already (`_DELIVER.controller.>`).
DeliverSubject: "_DELIVER." + ControllerName + ".merges",
Filters: []string{ControllerFollows[3]},
Push: true,
AckWaitSeconds: 30,
MaxAckPending: 1,
FromNow: true,
TakesOverFrom: ControllerName,
Why: "the forge's merges, one at a time; no max-deliver and never reset, because a merge is never " +
"dropped and no person releases one",
},
{
Name: ControllerName,
Stream: "EVENTS",
@@ -587,9 +581,6 @@ func NoticesConsumer() Consumer {
// one on DEAD_LETTER_NOTICES, which the keeper of dead letters asserts and retries by itself, so a fault
// there never stops the controller serving (novox/hq issue 330).
func AssertServingConsumers(e Ensurer) error {
if err := removeRetired(e); err != nil {
return err
}
for _, c := range MeshConsumers() {
if c.Stream == DeadLetterNoticesStream {
continue
@@ -612,9 +603,6 @@ type Ensurer interface {
// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the
// refusal names the stream rather than the order.
func AssertMeshConsumers(e Ensurer) error {
if err := removeRetired(e); err != nil {
return err
}
for _, c := range MeshConsumers() {
if err := e.EnsureConsumer(c); err != nil {
return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err)
+1 -18
View File
@@ -181,11 +181,6 @@ func Users(r Records) ([]Principal, error) {
}
}
for _, d := range r.Assigned[node] {
// **The node-engine's manifest is no placed module's** (novox/hq ADR 0563 §2, §3; sweep 2026-10-30, issue
// 774): the guard mesh-sdk #136 put in buscompose, kept here while the controller composes with its own copy.
if !Composed(d) {
continue
}
// **A process account is its own, delivered in its own secret** (broker-<process id>, novox/hq
// ADR 0436 §1, §2), never the module's `broker` — so it is composed whether or not the module
// itself has an account, and is never carried by the machine's runtime, which runs as the
@@ -232,7 +227,7 @@ func Users(r Records) ([]Principal, error) {
if runtimeHere {
var carried []Declared
for _, d := range r.Assigned[node] {
if CarriedByRunner(d) {
if d.RunsAs == "" && runnerServes(d) {
// Never a restricted stream for what the tool runner carries (novox/hq ADR 0376 §1).
d.Hears = ComposedHears(d)
carried = append(carried, d)
@@ -459,15 +454,3 @@ func declaredStreamsOf(r Records) []string {
sort.Strings(out)
return out
}
// EngineModule is the node-engine's manifest's module name (mesh-sdk buscompose.EngineModule).
const EngineModule = "mesh-host"
// Composed says a placed module is composed as one: its own user, its processes and what the runner carries of it. The
// node-engine's manifest is not, though a placement may name it: each node's engine speaks as the node's account, which
// alone writes the engine's state (ADR 0563 §2, §3); granted to another account it is a second writer, and the writers
// table refuses the whole list (issue 774: mesh-host_acts, then mesh-host_health, 2026-10-30). mesh-sdk buscompose.Composed.
func Composed(d Declared) bool { return d.Module != EngineModule }
// CarriedByRunner says the machine's tool runner carries this placed module (mesh-sdk buscompose.CarriedByRunner).
func CarriedByRunner(d Declared) bool { return Composed(d) && d.RunsAs == "" && runnerServes(d) }
+10 -34
View File
@@ -6,9 +6,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce"
"slices"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
)
@@ -121,18 +119,22 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) {
// announcement does not make readable is granted nothing, and said; an owner that announced nothing yet is read as
// before. Announcements that cannot be read are said, and every read is granted as before: the bus's users are
// composed either way.
//
// **Judged for its reader, by the SDK's judgement** (sweep 2026-10-30, issue 774): mesh-sdk buscompose's
// ReadsAnnouncedBy knows the named readers (ADR 0547 clause 9, ADR 0561 §3), so the lease is granted to the parts
// named for it and refused to every other. The controller's own reader-less copy refused every named reader: 2536
// refused reads of mesh-controller.lease on 2026-10-30, each said on every composition.
if Announcements != nil {
announced, err := Announcements(ctx)
if err != nil {
fmt.Printf("the states the modules announced cannot be read, so every read is granted from `reads` as before: %v\n", err)
} else {
out.Announced = announced
judgeReads(out.Assigned, announced, sayRefusedOnce)
for node, ds := range out.Assigned {
for k := range ds {
granted, refused := broker.ReadsAnnounced(ds[k].Reads, announced)
ds[k].Reads, ds[k].ReadsRefused = granted, refused
for _, r := range refused {
fmt.Printf("%s on %s reads %s, which its owner does not announce readable: not granted\n",
ds[k].Module, node, r)
}
}
}
}
}
@@ -613,29 +615,3 @@ func hearsOf(m catalogue.Manifest, declared map[string]catalogue.Manifest, node
}
return out, touches, onEvents
}
// refusalsSaid are the refused reads already said by this process, by `<module>@<node> <read>`: a refusal is said once,
// when first composed, not on every composition (about 420 times a read on 2026-10-30). `status` names them all.
var refusalsSaid sync.Map
// sayRefusedOnce says a refused read the first time this process composes it.
func sayRefusedOnce(module, node, read string) {
if _, said := refusalsSaid.LoadOrStore(module+"@"+node+" "+read, true); said {
return
}
fmt.Printf("%s on %s reads %s, which its owner does not make readable by it: not granted\n", module, node, read)
}
// judgeReads splits each assigned module's reads into granted and refused, for that module, by what the owners announced
// and the states' named readers (buscompose.ReadsAnnouncedBy), and says each refusal.
func judgeReads(assigned map[string][]broker.Declared, announced map[string]announce.States, say func(module, node, read string)) {
for node, ds := range assigned {
for k := range ds {
granted, refused := buscompose.ReadsAnnouncedBy(ds[k].Module, ds[k].Reads, announced)
ds[k].Reads, ds[k].ReadsRefused = granted, refused
for _, r := range refused {
say(ds[k].Module, node, r)
}
}
}
}
-44
View File
@@ -1,44 +0,0 @@
package inventory
import (
"slices"
"testing"
"git.novox.be/novox/mesh-sdk/go/announce"
"github.com/novox/mesh-controller/internal/broker"
)
// Sweep 2026-10-30 (issue 774): a read is judged for its reader. The controller's lease, which its announcement marks
// readable by nobody, is granted to the parts named for it (ADR 0561 §3) and refused to every other (mesh-config named since mesh-sdk ef1ec1c9); a state its owner
// announced readable is granted; one it keeps and did not is refused; an owner that announced nothing is read as before.
// Each refusal is said, and a refusal already said is not said again.
func TestAReadIsJudgedForItsReader(t *testing.T) {
announced := map[string]announce.States{
"mesh-controller": broker.ControllerAnnouncement("novox"),
"owner": {Module: "owner", States: []announce.State{{Name: "open", Readable: true}, {Name: "closed"}}},
}
for name, c := range map[string]struct {
reader, read string
granted bool
}{
"the lease, by the vault": {"mesh-vault", "mesh-controller.lease", true},
"the lease, by the grants part": {"mesh-grants", "mesh-controller.lease", true},
"the lease, by mesh-config": {"mesh-config", "mesh-controller.lease", true},
"the lease, by a module not named": {"restic", "mesh-controller.lease", false},
"a readable state": {"reader", "owner.open", true},
"a state kept and not readable": {"reader", "owner.closed", false},
"an owner that announced nothing": {"reader", "quiet.any", true},
"a mirror the controller marks open": {"pacman", "mesh-controller.declared-packages", true},
} {
assigned := map[string][]broker.Declared{"novox": {{Module: c.reader, Reads: []string{c.read}}}}
var said []string
judgeReads(assigned, announced, func(module, node, read string) { said = append(said, read) })
d := assigned["novox"][0]
if slices.Contains(d.Reads, c.read) != c.granted || slices.Contains(d.ReadsRefused, c.read) == c.granted {
t.Errorf("%s: granted %v, refused %v", name, d.Reads, d.ReadsRefused)
}
if (len(said) == 1) == c.granted {
t.Errorf("%s: said %v", name, said)
}
}
}
-11
View File
@@ -26,17 +26,6 @@ type ProvReport struct {
WaitsFor []string `json:"waits-for,omitempty"`
Templates string `json:"templates,omitempty"`
At time.Time `json:"at"`
// FailingSince is, on a failed report, when the module's standing failure began, kept by the client while only its
// words change (mesh-catalog 3aae6c08); zero from an older client.
FailingSince time.Time `json:"failing-since,omitzero"`
}
// Since is when a failed report's failure began: FailingSince when the client says it, else At.
func (r ProvReport) Since() time.Time {
if !r.FailingSince.IsZero() {
return r.FailingSince
}
return r.At
}
// The words a client's report says.
+36 -98
View File
@@ -127,13 +127,9 @@ type CallLog struct {
// keeper keeps every call beyond this process, when Durably was given one; writes go through
// one goroutine, in order, so a call's last state is the one kept.
keeper CallKeeper
holder string
// queued are the calls' states waiting for the keeper, one per call, in the order each call was first queued, under
// qmu (keep is called with mu held); woken is told when one is queued (callQueue).
qmu sync.Mutex
queued callQueue
woken chan struct{}
keeper CallKeeper
holder string
writes chan Call
logger *log.Logger
lost int // writes the keeper could not take, said once each
keepErr error
@@ -169,8 +165,8 @@ const keepTries = 5
func (l *CallLog) Durably(ctx context.Context, keeper CallKeeper, holder string, logger *log.Logger) error {
l.mu.Lock()
l.keeper, l.holder, l.logger = keeper, holder, logger
if l.woken == nil {
l.woken = make(chan struct{}, 1)
if l.writes == nil {
l.writes = make(chan Call, 256)
go l.keepWrites()
}
l.mu.Unlock()
@@ -207,109 +203,51 @@ func orSomebody(s string) string {
return s
}
// callQueueBound is how many calls may wait for the keeper at once: far beyond a burst of reads (about 200 calls in one
// second on 2026-10-30), and a bound, so a keeper that takes nothing for long is said rather than grown into.
const callQueueBound = 8192
// callQueue holds the latest state of each call waiting for the keeper. **One state per call** (sweep 2026-10-30, issue
// 774): a call's later state replaces its earlier one still waiting, since only the last is kept anyway; a burst of
// read calls queued begin and finish each, and a channel of 256 dropped 30 of them in one second.
type callQueue struct {
order []string
byID map[string]Call
}
// put queues c, replacing a waiting state of the same call; false when the queue is full.
func (q *callQueue) put(c Call) bool {
if q.byID == nil {
q.byID = map[string]Call{}
}
if _, waiting := q.byID[c.ID]; waiting {
q.byID[c.ID] = c
return true
}
if len(q.order) >= callQueueBound {
return false
}
q.order = append(q.order, c.ID)
q.byID[c.ID] = c
return true
}
// take is every waiting state, in order, and empties the queue.
func (q *callQueue) take() []Call {
out := make([]Call, 0, len(q.order))
for _, id := range q.order {
out = append(out, q.byID[id])
}
q.order, q.byID = nil, nil
return out
}
// keep queues one call's state for the keeper. Never blocks a call: a queue that is full is a keeper
// that is not taking writes, and that is said rather than waited on.
func (l *CallLog) keep(c Call) {
if l.woken == nil {
return
}
l.qmu.Lock()
queued := l.queued.put(c)
if !queued {
l.lost++
}
lost := l.lost
l.qmu.Unlock()
if !queued {
if l.logger != nil {
l.logger.Printf("%s (%s.%s) is kept in memory only: %d calls wait for the bus to take their records "+
"(%d not kept)", c.ID, c.Seat, c.Verb, callQueueBound, lost)
}
if l.writes == nil {
return
}
select {
case l.woken <- struct{}{}:
case l.writes <- c:
default:
l.lost++
if l.logger != nil {
l.logger.Printf("%s (%s.%s) is kept in memory only: the bus is not taking calls' records (%d not kept)",
c.ID, c.Seat, c.Verb, l.lost)
}
}
}
func (l *CallLog) keepWrites() {
for range l.woken {
l.qmu.Lock()
waiting := l.queued.take()
l.qmu.Unlock()
for _, c := range waiting {
l.keepOne(c)
}
}
}
// keepOne writes one call's state through the keeper, under the lease.
func (l *CallLog) keepOne(c Call) {
l.mu.Lock()
gate := l.epoch
l.mu.Unlock()
if gate != nil {
if _, err := gate(); err != nil {
if l.logger != nil {
l.logger.Printf("%s (%s.%s, %s) is not kept on the bus: %v — the controller holding the lease "+
"marks it abandoned", c.ID, c.Seat, c.Verb, c.State, err)
for c := range l.writes {
l.mu.Lock()
gate := l.epoch
l.mu.Unlock()
if gate != nil {
if _, err := gate(); err != nil {
if l.logger != nil {
l.logger.Printf("%s (%s.%s, %s) is not kept on the bus: %v — the controller holding the lease "+
"marks it abandoned", c.ID, c.Seat, c.Verb, c.State, err)
}
continue
}
return
}
}
var err error
for try := 0; try < keepTries; try++ {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err = l.keeper.Keep(ctx, c)
cancel()
if err == nil {
break
var err error
for try := 0; try < keepTries; try++ {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err = l.keeper.Keep(ctx, c)
cancel()
if err == nil {
break
}
time.Sleep(time.Duration(try+1) * time.Second)
}
if err != nil && l.logger != nil {
l.logger.Printf("%s (%s.%s, %s) could not be kept on the bus after %d tries: %v — `calls` "+
"answers it from memory until this controller stops", c.ID, c.Seat, c.Verb, c.State, keepTries, err)
}
time.Sleep(time.Duration(try+1) * time.Second)
}
if err != nil && l.logger != nil {
l.logger.Printf("%s (%s.%s, %s) could not be kept on the bus after %d tries: %v — `calls` "+
"answers it from memory until this controller stops", c.ID, c.Seat, c.Verb, c.State, keepTries, err)
}
}
+4 -32
View File
@@ -254,7 +254,8 @@ func TestAHandoverRefusalIsMarkedRetryable(t *testing.T) {
// who may call the seat, and a token is the one-time right to become a machine of the mesh.
func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
l.woken = make(chan struct{}, 1) // queued for the keeper, and nothing takes them: read below
writes := make(chan Call, 4)
l.writes = writes
l.serveCall("mesh-controller", "token", json.RawMessage(`{"new":"laptop"}`), "_INBOX.x.1",
func(context.Context, json.RawMessage) (any, error) { return "token for laptop: s3cret-join", nil },
a.respond, nil)
@@ -265,7 +266,8 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) {
if len(recent) != 1 || strings.Contains(string(recent[0].Answer), "s3cret-join") {
t.Fatalf("the token was kept in memory: %+v", recent)
}
for _, c := range l.queued.take() {
close(writes)
for c := range writes {
if strings.Contains(string(c.Answer), "s3cret-join") {
t.Fatalf("the token was sent to be kept on the bus: %s", c.Answer)
}
@@ -274,33 +276,3 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) {
// A verb whose handler panics answers an error, and the controller serves the next call (review of
// novox/hq issue 327: read verbs are answered in the serving process now).
// Sweep 2026-10-30 (issue 774): a burst of calls never loses a record while the keeper is slow: each call waits once,
// with its latest state, and every call of a burst far over the old queue of 256 is kept with its last state.
func TestABurstOfCallsIsKeptWithEachCallsLastState(t *testing.T) {
var q callQueue
for i := range 2000 {
id := fmt.Sprintf("call-%d", i)
if !q.put(Call{ID: id, State: CallRunning}) || !q.put(Call{ID: id, State: CallAnswered}) {
t.Fatalf("%s was refused by a queue of %d", id, callQueueBound)
}
}
got := q.take()
if len(got) != 2000 {
t.Fatalf("%d calls wait, want 2000", len(got))
}
for i, c := range got {
if c.ID != fmt.Sprintf("call-%d", i) || c.State != CallAnswered {
t.Fatalf("call %d waits as %+v", i, c)
}
}
if len(q.take()) != 0 {
t.Fatal("taken calls wait again")
}
for i := range callQueueBound {
q.put(Call{ID: fmt.Sprint(i)})
}
if q.put(Call{ID: "one too many"}) {
t.Fatal("the queue grew past its bound")
}
}
+3 -2
View File
@@ -30,8 +30,9 @@ var Contracts = map[string]Contract{
KindBuilt: {Unordered: "a build's outcome on the retired build seat's subjects: dropped, nothing takes a build in " +
"from the bus any more (R4g, ADR 0501 §1); the subjects stay followed while the installer's carried user list " +
"names them"},
KindSourceMoved: {Unordered: "the forge's merge on EVENTS: no longer followed, dropped if one arrives; the forge " +
"bridge's stream FORGE is the record of the merges, read at a moved module's registration (sweep 2026-10-30)"},
KindSourceMoved: {Ordered: "by the merge's commit against what was built from it: a merge already acted on " +
"or older than the last look is history, not a second plan (issues 250, 266)",
Tests: []string{"TestAMergeOlderThanTheLastLookIsHistory", "TestAMergeMatchesTheSourcesBuiltFromIt"}},
KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
KindHealth: {Ordered: "by the time the node-engine looked (`at`, on the machine's clock): a statement older " +
"than the one kept for that machine is refused, so an event arriving after a newer report does not undo it " +
+4 -2
View File
@@ -125,7 +125,8 @@ func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnsw
// bus, where `calls` answers anyone who may call the seat.
func TestAMeshCLIRecordKeepsNeitherItsLineNorItsAnswerOnTheBus(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
l.woken = make(chan struct{}, 1) // queued for the keeper, and nothing takes them: read below
writes := make(chan Call, 4)
l.writes = writes
asked, _ := json.Marshal(CLIAsked{Line: []string{"settings", "set", "x", `{"password":"s3cret"}`}, Account: "op"})
l.serveCall(CLISeat, "laptop", asked, "_INBOX.node.laptop.abcdefghijklmnopqrstuv",
func(context.Context, json.RawMessage) (any, error) {
@@ -133,7 +134,8 @@ func TestAMeshCLIRecordKeepsNeitherItsLineNorItsAnswerOnTheBus(t *testing.T) {
},
a.respond, nil)
_ = a.only()
for _, c := range l.queued.take() {
close(writes)
for c := range writes {
if carries(c.Args, "s3cret") || carries(c.Answer, "s3cret-join") {
t.Fatalf("the bus was sent %s / %s", c.Args, c.Answer)
}
-3
View File
@@ -52,9 +52,6 @@ type MoveSecretAnswer struct {
Secret string `json:"secret,omitempty"`
Generation int `json:"generation,omitempty"`
Unchanged bool `json:"unchanged,omitempty"`
// Superseded says the vault holds another value of the secret, which is the live one, and this machine's older value
// was not moved and is not asked again (mesh-host b9c24a7, sweep 2026-10-30).
Superseded bool `json:"superseded,omitempty"`
}
// MoveSecretWithin is how long the controller waits for one move: the engine asks the vault twice.
+2 -3
View File
@@ -30,9 +30,8 @@ func TestTheMovesAskAndAnswerKeepTheirFieldNames(t *testing.T) {
"provision sealed sealed_to" {
t.Fatalf("the ask's fields are %q", got)
}
answer := MoveSecretAnswer{Said: "s", Refused: "r", Secret: "own/a/b/c", Generation: 1, Unchanged: true,
Superseded: true}
if got := keys(answer); got != "generation refused said secret superseded unchanged" {
answer := MoveSecretAnswer{Said: "s", Refused: "r", Secret: "own/a/b/c", Generation: 1, Unchanged: true}
if got := keys(answer); got != "generation refused said secret unchanged" {
t.Fatalf("the answer's fields are %q", got)
}
if broker.AskMoveSecretSubject("laptop") != "mesh.node.laptop.ask.move-secret" ||
+20 -6
View File
@@ -62,7 +62,7 @@ func Nats(js *broker.JetStream) Inbound {
// whatever was asked for — and not at all when nothing was.
func (n *natsInbound) Also(kind string) error {
switch kind {
case KindModuleMoved, KindCatchUp, KindProvisioner, KindPullUpdated, KindDecided,
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated, KindDecided,
KindDeliveryMerged:
n.follows[kind] = true
return nil
@@ -145,6 +145,24 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
defer func() { _ = followed.Unsubscribe() }()
}
// And the forge's merges, off a consumer of their own: no max-deliver and no reset, so a merge held for the
// plans waits for them however long (novox/hq ADR 0407 decision 3). Bound to the same channel: one loop.
if n.follows[KindSourceMoved] {
if events == nil {
events = make(chan *nats.Msg, Prefetch)
}
merges, err := standingBy(ctx, log.Default(), "EVENTS", func() (*nats.Subscription, error) {
return js.ChanSubscribe(MergedSubject, events, nats.Bind("EVENTS", broker.MergesConsumerName))
})
if err != nil {
return fmt.Errorf("subscribing to the forge's merges: %w", err)
}
if merges == nil {
return nil
}
defer func() { _ = merges.Unsubscribe() }()
}
// A connection that dropped is said, not discovered. A controller whose bus connection is gone
// is a mesh where nothing can be told anything.
gone := make(chan error, 1)
@@ -405,11 +423,7 @@ func (m *natsControl) Hold(after time.Duration) error {
if _, already := m.on.since[m.seq]; !already {
m.on.since[m.seq] = time.Now()
}
if err := m.msg.NakWithDelay(after); err != nil {
return err
}
Loop.HeldUntil(time.Now().Add(after))
return nil
return m.msg.NakWithDelay(after)
}
func (m *natsControl) forget() {
@@ -394,3 +394,35 @@ func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) {
case <-time.After(3 * time.Second):
}
}
func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) {
first, most := MergeHeldFirst, MergeHeldMost
MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond
t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most })
js := aBus(t)
up := &manyTimesBusy{busy: 8}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(up); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the merges consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil
})
body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"})
if _, err := js.Context().Publish(MergedSubject, body); err != nil {
t.Fatal(err)
}
eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 })
eventually(t, "the merge being acknowledged", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil && info.NumAckPending == 0 && info.NumPending == 0
})
if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 {
t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs)
}
}
+44
View File
@@ -2,6 +2,7 @@ package link
import (
"context"
"errors"
"io"
"log"
"sync"
@@ -198,3 +199,46 @@ func (s slowly) Heard(context.Context, Report) (bool, error) { s.work(); return
type sameAgain struct{}
func (sameAgain) Heard(context.Context, Report) (bool, error) { return false, nil }
// manyTimesBusy is an upgrader whose plans are busy for its first merges, as another controller's hold is.
type manyTimesBusy struct {
toldAbout
busy int
calls int
}
func (u *manyTimesBusy) SourceMoved(context.Context, SourceMoved) error {
u.mu.Lock()
defer u.mu.Unlock()
u.calls++
if u.calls <= u.busy {
return errors.Join(ErrTryAgain, errors.New("another controller is working the plans"))
}
return nil
}
func (u *manyTimesBusy) called() int {
u.mu.Lock()
defer u.mu.Unlock()
return u.calls
}
// **A merge held more than five times is still handed over and recorded** (issue 973, novox/hq ADR 0407 decision
// 3): the merges have a durable consumer of their own with no max-deliver, so they never go to DEAD_LETTERS to wait
// for a person; the delay grows, and is capped.
// The delay a held merge waits grows from the first to the most and stays there.
func TestAHeldMergesDelayGrowsAndIsCapped(t *testing.T) {
var got []time.Duration
for _, held := range []time.Duration{0, 4 * time.Second, 10 * time.Second, 25 * time.Second, 70 * time.Second,
200 * time.Second, 20 * time.Minute} {
got = append(got, MergeRedeliverAfter(held))
}
want := []time.Duration{5 * time.Second, 5 * time.Second, 10 * time.Second, 25 * time.Second, 70 * time.Second,
200 * time.Second, 5 * time.Minute}
for i := range want {
if got[i] != want[i] {
t.Errorf("held %d: waits %v; want %v", i, got[i], want[i])
}
}
}
+61 -3
View File
@@ -49,6 +49,8 @@ type Upgrader interface {
// stop every upgrade behind it — except the store unreachable for the moment, which is asked
// again for a bounded time (novox/hq issue 083).
Upgraded(ctx context.Context, u Upgraded) error
// SourceMoved is a merge on the forge: build what that source produces, bases first.
SourceMoved(ctx context.Context, m SourceMoved) error
}
// Checker is what the controller does when the forge says a pull request's head moved: ask for it to be
@@ -103,6 +105,9 @@ const GiveUpAfter = 2 * time.Minute
// Follows says what to do about upgrades, and asks for them to be delivered.
func (s *Server) Follows(u Upgrader) error {
if err := s.inbound.Also(KindSourceMoved); err != nil {
return err
}
if err := s.inbound.Also(KindModuleMoved); err != nil {
return err
}
@@ -220,9 +225,7 @@ func (s *Server) act(ctx context.Context, m Control) {
case KindModuleMoved:
s.moved(ctx, m)
case KindSourceMoved:
// The forge's merges are no longer followed (sweep 2026-10-30): the forge bridge's stream is their record, read
// where a merge is needed (keptForgeMerge). One that arrives on an old consumer is let go of.
_ = m.Drop()
s.sourceMoved(ctx, m)
case KindCatchUp:
s.catchingUp(ctx, m)
case KindProvisioner:
@@ -518,6 +521,61 @@ func short(commit string) string {
return commit
}
// MergeHeldFirst is how long a merge the controller could not record yet waits before the bus hands it over
// again, and MergeHeldMost the longest it ever waits: the delay grows with how long the merge has been held, from
// the first to the most (novox/hq ADR 0407 decision 3). Variables so a test need not wait.
var (
MergeHeldFirst = 5 * time.Second
MergeHeldMost = 5 * time.Minute
)
// MergeRedeliverAfter is the delay for a merge held for the time given: that time, so the delay about doubles
// with each hold, between MergeHeldFirst and MergeHeldMost.
func MergeRedeliverAfter(heldFor time.Duration) time.Duration {
return min(max(heldFor, MergeHeldFirst), MergeHeldMost)
}
// sourceMoved acts on the forge's announcement of a merge. **A merge is never dropped** (issue 921, novox/hq ADR
// 0407 decision 3): one the controller could not record for now (the plans held by another controller, the store
// away: ErrTryAgain) is held and handed over again until it is recorded; the controller giving up on it would
// lose a change for good. Anything else is an answer and is taken: a build that fails is reported by the build
// itself, and re-delivering the merge would only re-fail it.
func (s *Server) sourceMoved(ctx context.Context, m Control) {
var moved SourceMoved
if err := json.Unmarshal(m.Body(), &moved); err != nil {
s.log.Printf("a merge announcement could not be read: %v", err)
_ = m.Took()
return
}
if moved.Commit == "" || moved.Repo == "" {
s.log.Printf("a merge announcement named no repository or no commit; ignored")
_ = m.Took()
return
}
err := s.upgrader.SourceMoved(ctx, moved)
// Left unsettled when the controller is ending, whatever error that gave (the plans' hold returns a bare
// context.Canceled): the bus hands it to whatever consumes next, and taking it would lose the merge.
if err != nil && ctx.Err() != nil {
return
}
if err != nil && errors.Is(err, ErrTryAgain) {
// Also left unsettled when the bus will not hold it.
delay := MergeRedeliverAfter(m.HeldFor())
if held := m.Hold(delay); held != nil {
s.log.Printf("%s/%s moved to %.8s and could not be held (%v): %v; left for the bus to hand over again",
moved.Owner, moved.Repo, moved.Commit, held, err)
return
}
s.log.Printf("%s/%s moved to %.8s and is held: %v; handed over again in %s", moved.Owner, moved.Repo,
moved.Commit, err, delay)
return
}
if err != nil {
s.log.Printf("%s/%s moved to %.8s and the mesh could not act on it: %v", moved.Owner, moved.Repo, moved.Commit, err)
}
_ = m.Took()
}
// pullUpdated asks for a pull request's merge check. Taken whatever happens: a check that could not be
// asked is said here, and the next push to the pull request asks again.
func (s *Server) pullUpdated(ctx context.Context, m Control) {
+60
View File
@@ -0,0 +1,60 @@
package link
import (
"context"
"encoding/json"
"errors"
"io"
"log"
"testing"
"time"
)
// heldControl is a merge announcement that records how it was settled.
type heldControl struct {
body []byte
took, held, drops int
}
func (c *heldControl) Kind() string { return "source-moved" }
func (c *heldControl) Body() []byte { return c.body }
func (c *heldControl) Subject() string { return "" }
func (c *heldControl) Redelivered() bool { return false }
func (c *heldControl) HeldFor() time.Duration { return 0 }
func (c *heldControl) Answer(context.Context, []byte) error { return nil }
func (c *heldControl) Took() error { c.took++; return nil }
func (c *heldControl) Hold(time.Duration) error { c.held++; return nil }
func (c *heldControl) Drop() error { c.drops++; return nil }
type sourceMover struct {
Upgrader
answer error
}
func (s sourceMover) SourceMoved(context.Context, SourceMoved) error { return s.answer }
// **A merge is acknowledged only after the controller recorded it** (novox/hq ADR 0407 decision 3, issue 1092): an
// answer of "not now" — the plan it belongs to could not be saved, the bus away, the lease lost — leaves the announcement
// unacknowledged, held to be handed over again; only an answer that is not "not now" settles it.
func TestAMergeTheControllerCouldNotRecordIsHeldAndNotAcknowledged(t *testing.T) {
body, err := json.Marshal(SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1"})
if err != nil {
t.Fatal(err)
}
s := &Server{log: log.New(io.Discard, "", 0)}
for name, tc := range map[string]struct {
answer error
took, held int
}{
"recorded": {nil, 1, 0},
"not now": {errors.Join(ErrTryAgain, errors.New("the plan is not saved")), 0, 1},
"an answer, not a wait": {errors.New("the mesh could not act on it"), 1, 0},
} {
s.upgrader = sourceMover{answer: tc.answer}
m := &heldControl{body: body}
s.sourceMoved(context.Background(), m)
if m.took != tc.took || m.held != tc.held || m.drops != 0 {
t.Errorf("%s: acknowledged %d times and held %d times, want %d and %d", name, m.took, m.held, tc.took, tc.held)
}
}
}
+60 -1
View File
@@ -17,7 +17,8 @@ func (r recordsWith) Built(context.Context, BuildResult) error { return r.err }
type upgradesWith struct{ err error }
func (u upgradesWith) Upgraded(context.Context, Upgraded) error { return u.err }
func (u upgradesWith) Upgraded(context.Context, Upgraded) error { return u.err }
func (u upgradesWith) SourceMoved(context.Context, SourceMoved) error { return u.err }
type replaysWith struct{ err error }
@@ -68,3 +69,61 @@ func TestACatchUpWaitsOutARestartingStore(t *testing.T) {
}
}
}
// A merge the controller could not record for now (the plans held by another controller, the store away) is held
// and handed over again, never taken: the bus keeps it until it is recorded (issue 921). Any other failure is an
// answer and is taken, as before.
func TestAMergeThatCouldNotBeRecordedNowIsHeldNotDropped(t *testing.T) {
merged := SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}
for _, c := range []struct {
what string
err error
want func(*settled) bool
}{
{"the plans held by another controller", errors.Join(ErrTryAgain, errors.New("another controller is working the plans")),
func(s *settled) bool { return s.unsettled() }},
{"the store away", errors.Join(ErrTryAgain, restarting), func(s *settled) bool { return s.unsettled() }},
{"cannot act", errors.New("no such module"), func(s *settled) bool { return s.acked && !s.nacked }},
{"kept", nil, func(s *settled) bool { return s.acked && !s.nacked }},
} {
s, in := serving()
s.upgrader = upgradesWith{err: c.err}
to := &settled{}
held := in.sends(t, to, KindSourceMoved, merged).(*fakeControl)
s.act(context.Background(), held)
if !c.want(to) {
t.Errorf("%s: a merge was settled as %+v", c.what, *to)
}
if c.err != nil && errors.Is(c.err, ErrTryAgain) && len(in.held) != 1 {
t.Errorf("%s: the merge was not held to be handed over again", c.what)
}
}
}
// A merge being held when the controller is replaced (its context ended) or when the bus will not hold it is left
// unsettled, never taken: the bus hands it to whatever consumes next (review of issue 921).
func TestAMergeIsLeftUnsettledWhenTheControllerEndsOrTheHoldFails(t *testing.T) {
merged := SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}
try := errors.Join(ErrTryAgain, errors.New("another controller is working the plans"))
// The plans' hold returns a bare context.Canceled when the controller is replaced, not ErrTryAgain.
s, in := serving()
s.upgrader = upgradesWith{err: context.Canceled}
to := &settled{}
ended, end := context.WithCancel(context.Background())
end()
s.act(ended, in.sends(t, to, KindSourceMoved, merged))
if !to.unsettled() {
t.Errorf("a merge was settled as %+v when the controller's context ended", *to)
}
s, in = serving()
s.upgrader = upgradesWith{err: try}
to = &settled{}
m := in.sends(t, to, KindSourceMoved, merged).(*fakeControl)
m.holdFails = errors.New("the bus will not hold it")
s.act(context.Background(), m)
if !to.unsettled() {
t.Errorf("a merge was settled as %+v when it could not be held", *to)
}
}
-20
View File
@@ -85,26 +85,6 @@ func nodeOfToolsAlive(subject string) (string, bool) {
type LoopActivity struct {
mu sync.Mutex
took time.Time
// due is when the latest message the loop held back with a delay is handed over again: until then the loop waits
// by its own choice, which is not deafness (sweep 2026-10-30: a merge held five minutes at a time read as a deaf
// controller every time its delay outran the deaf bound).
due time.Time
}
// HeldUntil records that the loop handed a message back to be delivered again at due.
func (l *LoopActivity) HeldUntil(due time.Time) {
l.mu.Lock()
defer l.mu.Unlock()
if due.After(l.due) {
l.due = due
}
}
// Due is when the latest message the loop held is handed over again; zero when it held none.
func (l *LoopActivity) Due() time.Time {
l.mu.Lock()
defer l.mu.Unlock()
return l.due
}
// Loop is this process's event loop.
+14
View File
@@ -90,3 +90,17 @@ A small Go program beside the proxy (`cmd/route-provider`) runs the SDK's provid
and the offer says `"answered-by": "helper"`: a handed consumer (gitea's origins file) is filled by its machine's client
from this module's entry, which carries the public names the controller composed for it. The module is **not** marked
moved: its own files, `acme.env` and the broker secret, are composed as before, and the proxy itself is unchanged.
## What it says on the bus, and its tools (issue 774)
The proxy keeps two states, announced readable (`conn.Keeps`, novox/hq ADR 0495), each keyed `<node>.<name>`:
| state | one key per | holds | written when |
|---|---|---|---|
| `routes` | served name | the rules it is served by (path, module, node, target, refusal, redirect, authentication); its certificate (authority, issuer, not-after, last renewal, outcome, or why it failed); the last upstream answer (status, time, error, path) | a route arrives, changes or leaves; an authority stores a certificate; a handshake fails to find one or hands out one not yet recorded; an upstream answer differs from the last one written, or the same one is a minute old |
| `unrouted` | name answered "no route for", in a request or a TLS handshake, under a domain the proxy serves | when it was last asked, how often since the start, whether it is an internal-only name asked from outside; never who asked (the journal keeps that, for fail2ban) | first asked, at most five new names a minute, then at most once a minute; deleted when the name is served; kept a week. Every other refusal is only counted, in one key per proxy (`<node>._other`: the last minute's count and the total), written once a minute. A refused write is dropped, never retried |
Nothing is written on a timer. The provider helper's binary serves the module's tools from those states, one call each:
`route_proxy_routes` (every served name with its target, certificate expiry and last answer), `route_proxy_route`
(one name in full, or that it is not served and when it was last asked for) and `route_proxy_unrouted` (the names
refused, newest first).
+11 -3
View File
@@ -1,6 +1,6 @@
// route-provider: route-proxy's Go half, one binary the node's runtime launches and speaks MCP to over stdio
// through the Go SDK. It serves no tools; it runs the SDK's provider helper that answers the consumers of `route`
// (issue 792). The TypeScript adapter beside it keeps writing the proxy's files.
// through the Go SDK. It runs the SDK's provider helper that answers the consumers of `route` (issue 792), and serves
// the module's tools, which read what the proxy keeps on the bus (issue 774, tools.go).
//
// stdout is the MCP channel, so everything this module says, it says on stderr.
package main
@@ -9,9 +9,12 @@ import (
"context"
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/provider"
"route-proxy/internal/routestate"
)
func say(format string, args ...any) {
@@ -25,12 +28,17 @@ func main() {
} else {
p := newProvider(node, provider.StateBus(ModuleName), say)
go func() {
// The helper's watches go through the runtime's channel, which Serve holds: one asked before it does is
// refused ("before Serve"), Run returns and the helper stays stopped until the process restarts.
<-stdio.Served()
if err := p.Run(context.Background()); err != nil {
say("the provider helper stopped: %v", err)
}
}()
}
if err := stdio.Serve("", nil); err != nil {
tools := front{routes: stdio.State(routestate.RoutesState), unrouted: stdio.State(routestate.UnroutedState),
now: time.Now}.tools()
if err := stdio.Serve("", tools); err != nil {
say("%v", err)
os.Exit(1)
}
@@ -9,7 +9,9 @@ import (
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// bucketBus is the helper's bus over in-memory buckets.
@@ -44,33 +46,60 @@ func (b *bucketBus) Delete(_ context.Context, bucket, key string) error {
return nil
}
// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792):
// the entry's values are the controller's composition, copied unchanged, and no vault is asked.
// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792),
// from the seat mesh-provisioner's plans and keys (novox/hq ADR 0570 clause 2), never the controller's assignments: a
// module not handed whose route the seat keys (grafana, issue 774) has a plan of provisions alone, and a helper that
// read only the assignments left its key asked for good, so the assigner withheld every new build of 38 such modules.
func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) {
bus := &bucketBus{buckets: map[string]map[string][]byte{}}
plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{
"provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox",
"provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea",
"names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal",
"name-web": "git.example.org"}}}}})
if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil {
t.Fatal(err)
for module, names := range map[string]map[string]string{
"gitea": {"name": "git.example.org", "internal-name": "git.novox.internal", "name-web": "git.example.org"},
"grafana": {"name": "grafana.example.org"},
} {
plan, err := provisions.EncodePlan(provisions.Plan{Node: "g14", Module: module, Plan: catalogue.Plan{
Provisions: []catalogue.PlanProvision{{Provision: ProvisionName, Provider: "novox", ProviderModule: ModuleName,
At: "10.7.0.1", From: "novox", As: "mesh_g14_" + module, Names: names}}}})
if err != nil {
t.Fatal(err)
}
key := "g14." + module
if err := bus.Put(context.Background(), provisions.PlansBucket, key, plan); err != nil {
t.Fatal(err)
}
asked, err := json.Marshal(provisions.Provision{V: provisions.Version, Provision: ProvisionName,
Consumer: provisions.End{Node: "g14", Module: module}, Provider: provisions.End{Node: "novox", Module: ModuleName},
State: provisions.Asked})
if err != nil {
t.Fatal(err)
}
if err := bus.Put(context.Background(), provisions.ProvisionsBucket, key+"."+ProvisionName, asked); err != nil {
t.Fatal(err)
}
}
p := newProvider("novox", bus, t.Logf)
out, err := p.Pass(context.Background())
if err != nil {
t.Fatal(err)
}
if len(out) != 1 || out[0].Status != provider.Answered {
if len(out) != 2 || out[0].Status != provider.Answered || out[1].Status != provider.Answered {
t.Fatalf("the pass did %+v", out)
}
var e provider.Entry
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)]["g14.gitea.route"], &e); err != nil {
t.Fatal(err)
}
if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" ||
e.Values["internal-name"] != "git.novox.internal" || e.VaultName != "" || e.User != "" || e.Refused != "" {
t.Fatalf("the entry is %+v", e)
for key, want := range map[string]map[string]string{
"g14.gitea.route": {"name-web": "git.example.org", "name": "git.example.org", "internal-name": "git.novox.internal"},
"g14.grafana.route": {"name": "grafana.example.org"},
} {
var e provider.Entry
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)][key], &e); err != nil {
t.Fatalf("%s: %v", key, err)
}
for k, v := range want {
if e.Values[k] != v {
t.Errorf("%s says %s %v, want %s", key, k, e.Values[k], v)
}
}
if e.State != provisions.Provided || e.VaultName != "" || e.User != "" || e.Refused != "" {
t.Errorf("the entry %s is %+v", key, e)
}
}
}
@@ -95,7 +124,8 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) {
if len(provides) != 1 || provides[0].(map[string]any)["answered-by"] != "helper" {
t.Fatalf("route is not said to be answered by the helper: %v", provides)
}
for _, k := range []string{"build", "provides", "state", "reads"} {
// Its states and its tools are what it says on the bus, not what it runs or writes (issue 774).
for _, k := range []string{"build", "provides", "state", "reads", "tools", "replaces"} {
delete(m, k)
}
pinned, err := json.Marshal(m) // sorted keys, compact
@@ -0,0 +1,296 @@
package main
// route-proxy's tools (issue 774): the web front's command line. Each answers from what the proxy keeps on the bus
// (route-proxy_routes and route-proxy_unrouted, written on change by the proxy itself), in one call, for every
// machine running a proxy: which sites are served and where they go, whether their certificates are good, what the
// last request through each got back, and which names were asked for and are served nowhere.
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// The tools' names, as the manifest declares them.
const (
RoutesTool = "route_proxy_routes"
RouteTool = "route_proxy_route"
UnroutedTool = "route_proxy_unrouted"
)
// reader is one of the module's states, as stdio.KeptState reads it.
type reader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// front is what the tools read: the routes and the refusals, and the clock a certificate's days left are counted by.
type front struct {
routes, unrouted reader
now func() time.Time
}
// RouteLine is one served name in the list: where it goes, how long its certificate holds, what it last answered.
type RouteLine struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
Reach string `json:"reach"`
Targets []string `json:"targets"`
// Reachable is yes when the last answer upstream gave was below 500, no when it was 500 or more or none came,
// unknown before any request.
Reachable string `json:"reachable"`
// Certificate is its outcome, issuer and expiry in words; ExpiresInDays is the whole days until not-after.
Certificate string `json:"certificate"`
ExpiresInDays *int `json:"expires_in_days,omitempty"`
LastAnswer *routestate.Answer `json:"last_answer,omitempty"`
}
func (f front) tools() []stdio.Tool {
node := map[string]any{"type": "string", "description": "only the proxy on this machine; every machine when left out"}
return []stdio.Tool{
{Name: RoutesTool,
Description: "Every site (web name, host name) the mesh's web front serves through route-proxy, the reverse proxy " +
"on the public machine: for each route the module and node it forwards to, its TLS certificate's issuer and " +
"expiry (days left), and the last answer upstream gave (status, time, error), so is X reachable and is its " +
"certificate good is answered without curl. Read from the proxy's routes state on the bus. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
Run: func(args map[string]any) (any, error) {
at, _ := args["node"].(string)
return f.list(at)
}},
{Name: RouteTool,
Description: "One site the mesh's web front (route-proxy) is asked about, in full: every rule it is served by " +
"(path, module, node, target, refusal, redirect, authentication), its certificate (issuer, not-after, last " +
"renewal and its outcome, or why it failed) and the last upstream answer. A name with no route says so, and " +
"when it was last asked for. Answers is X reachable, why does this site not load. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
"name": map[string]any{"type": "string", "description": "the site's name, such as app.example.org"},
"node": node}},
Run: func(args map[string]any) (any, error) {
name, _ := args["name"].(string)
at, _ := args["node"].(string)
return f.one(name, at)
}},
{Name: UnroutedTool,
Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " +
"a TLS handshake that no route serves, newest first, with when it was last asked and how often. Only names under a " +
"domain the proxy serves are kept by name, a few new ones a minute; every other refusal (a scan) is counted per " +
"proxy, the last minute's count and the total since its start. A site that should be served and is missing " +
"its route shows here. Kept a week. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
Run: func(args map[string]any) (any, error) {
at, _ := args["node"].(string)
return f.refusals(at)
}},
}
}
// read is every value of a state, of node's proxy alone when node is named, decoded into a fresh T each.
func read[T any](state reader, node string) ([]T, error) {
keys, err := state.Keys()
if err != nil {
return nil, err
}
var out []T
for _, key := range keys {
at, _ := routestate.Split(key)
if node != "" && at != node {
continue
}
entry, err := state.Get(key)
if err != nil {
return nil, err
}
if entry == nil {
continue // deleted between the list and the read
}
var v T
if err := json.Unmarshal(entry.Value, &v); err != nil {
return nil, fmt.Errorf("%s holds what is not a route-proxy record: %w", key, err)
}
out = append(out, v)
}
return out, nil
}
func (f front) list(node string) (map[string]any, error) {
routes, err := read[routestate.Route](f.routes, node)
if err != nil {
return nil, err
}
lines := make([]RouteLine, 0, len(routes))
unreachable, expiring := 0, 0
for _, r := range routes {
line := f.line(r)
if line.Reachable == "no" {
unreachable++
}
if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == routestate.Failed {
expiring++
}
lines = append(lines, line)
}
sort.Slice(lines, func(i, j int) bool {
if lines[i].Name != lines[j].Name {
return lines[i].Name < lines[j].Name
}
return lines[i].Proxy < lines[j].Proxy
})
return map[string]any{"routes": lines, "served": len(lines), "unreachable": unreachable,
"certificates_failing_or_within_14_days": expiring}, nil
}
func (f front) line(r routestate.Route) RouteLine {
line := RouteLine{Name: r.Name, Proxy: r.Proxy, Reach: r.Reach, LastAnswer: r.LastAnswer, Reachable: "unknown",
Targets: []string{}}
for _, t := range r.Targets {
line.Targets = append(line.Targets, describe(t))
}
if a := r.LastAnswer; a != nil {
line.Reachable = "yes"
if a.Error != "" || a.Status >= 500 || a.Status == 0 {
line.Reachable = "no"
}
}
c := r.Certificate
words := []string{}
if c.Outcome != "" {
words = append(words, c.Outcome)
}
if c.Issuer != "" {
words = append(words, "by "+c.Issuer)
}
if c.NotAfter != nil {
days := int(c.NotAfter.Sub(f.now()).Hours() / 24)
line.ExpiresInDays = &days
words = append(words, "until "+c.NotAfter.UTC().Format("2006-01-02"))
}
if c.Error != "" {
words = append(words, "("+c.Error+")")
}
if len(words) == 0 {
words = append(words, "none seen yet")
}
line.Certificate = c.Authority + ": " + strings.Join(words, " ")
return line
}
// describe is one rule in a few words: path → module@node at target, or what it answers with.
func describe(t routestate.Target) string {
path := t.Path
if path == "" {
path = "/"
}
var what string
switch {
case t.Deny:
what = "refused"
case t.Redirect != "":
what = "redirected to " + t.Redirect
default:
what = fmt.Sprintf("%s@%s at %s", t.Module, t.Node, t.Target)
}
if t.Sealed != "" {
what += " (authentication cannot be checked: " + t.Sealed + ")"
} else if t.Auth {
what += " (authenticated)"
}
return path + " → " + what
}
func (f front) one(name, node string) (map[string]any, error) {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return nil, fmt.Errorf("name is the site to read, such as app.example.org")
}
routes, err := read[routestate.Route](f.routes, node)
if err != nil {
return nil, err
}
var found []routestate.Route
for _, r := range routes {
if r.Name == name {
found = append(found, r)
}
}
if len(found) > 0 {
out := map[string]any{"name": name, "routed": true, "routes": found}
if len(found) == 1 {
out["reachable"] = f.line(found[0]).Reachable
}
return out, nil
}
asked, _, err := f.asked(node)
if err != nil {
return nil, err
}
for _, a := range asked {
if a.Name == name {
return map[string]any{"name": name, "routed": false, "asked": a,
"says": fmt.Sprintf("no proxy serves %s; the proxy on %s answered \"no route\" for it, last at %s",
name, a.Proxy, a.AskedAt.UTC().Format(time.RFC3339))}, nil
}
}
return map[string]any{"name": name, "routed": false,
"says": fmt.Sprintf("no proxy serves %s, and none was asked for it this week", name)}, nil
}
func (f front) refusals(node string) (map[string]any, error) {
asked, others, err := f.asked(node)
if err != nil {
return nil, err
}
sort.Slice(asked, func(i, j int) bool { return asked[i].AskedAt.After(asked[j].AskedAt) })
if asked == nil {
asked = []routestate.Unrouted{}
}
return map[string]any{"unrouted": asked, "names": len(asked), "other_refusals": others}, nil
}
// asked is the names refused and kept by name, and each proxy's count of the refusals not kept by name.
func (f front) asked(node string) ([]routestate.Unrouted, []routestate.Other, error) {
keys, err := f.unrouted.Keys()
if err != nil {
return nil, nil, err
}
names, others := keySet{}, keySet{}
for _, key := range keys {
if _, name := routestate.Split(key); name == routestate.OtherName {
others[key] = true
} else {
names[key] = true
}
}
asked, err := read[routestate.Unrouted](only{f.unrouted, names}, node)
if err != nil {
return nil, nil, err
}
counted, err := read[routestate.Other](only{f.unrouted, others}, node)
if counted == nil {
counted = []routestate.Other{}
}
return asked, counted, err
}
// keySet is a set of keys; only is a state narrowed to them.
type keySet map[string]bool
type only struct {
reader
keys keySet
}
func (o only) Keys() ([]string, error) {
out := make([]string, 0, len(o.keys))
for k := range o.keys {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
@@ -0,0 +1,99 @@
package main
import (
"encoding/json"
"sort"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// held is a state as the bus holds it, read-only.
type held map[string]any
func (h held) Keys() ([]string, error) {
out := []string{}
for k := range h {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (h held) Get(key string) (*stdio.StateEntry, error) {
v, ok := h[key]
if !ok {
return nil, nil
}
raw, err := json.Marshal(v)
return &stdio.StateEntry{Key: key, Value: raw}, err
}
// The web front's tools answer from the bus, in one call each (issue 774): every site with its target, certificate
// expiry and last answer; one site in full, or that it is not served and when it was asked for; and the names the
// proxy answered "no route for".
func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) {
now := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
expires := now.Add(30 * 24 * time.Hour)
routes := held{
"anchor.grafana.example": routestate.Route{Name: "grafana.example", Proxy: "anchor", Reach: routestate.ReachPublic,
Targets: []routestate.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}},
Certificate: routestate.Certificate{Authority: routestate.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: routestate.Issued},
LastAnswer: &routestate.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}},
"anchor.drive.example": routestate.Route{Name: "drive.example", Proxy: "anchor", Reach: routestate.ReachPublic,
Targets: []routestate.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}},
Certificate: routestate.Certificate{Authority: routestate.ReachPublic},
LastAnswer: &routestate.Answer{Status: 200, At: now}},
}
unrouted := held{
"anchor.old.example": routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3},
"anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1},
"anchor._other": routestate.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900},
}
tools := map[string]stdio.Tool{}
for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() {
tools[tool.Name] = tool
}
for _, row := range []struct {
tool string
args map[string]any
want []string // what the answer says, in its JSON
}{
{RoutesTool, nil, []string{`"served":2`, `"unreachable":1`,
`"name":"grafana.example"`, `"reachable":"no"`, `"expires_in_days":30`, `/ → grafana@home-server at http://10.0.0.2:3000`,
`"name":"drive.example"`, `"reachable":"yes"`, `public: none seen yet`}},
{RoutesTool, map[string]any{"node": "home-server"}, []string{`"served":0`}},
{RouteTool, map[string]any{"name": "Grafana.Example"}, []string{`"routed":true`, `"reachable":"no"`,
`connection refused`, `"issuer":"Let's Encrypt R11"`}},
{RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`,
`answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}},
{RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}},
{UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`,
`"other_refusals":[{"proxy":"anchor"`, `"last_minute":40`, `"since_start":900`}},
} {
t.Run(row.tool, func(t *testing.T) {
tool, ok := tools[row.tool]
if !ok {
t.Fatalf("%s is not served", row.tool)
}
if !strings.Contains(tool.Description, "route-proxy") {
t.Errorf("%s's description does not name the web front, so a search does not find it", row.tool)
}
answer, err := tool.Run(row.args)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(answer)
for _, w := range row.want {
if !strings.Contains(string(raw), w) {
t.Errorf("%s(%v) does not say %s:\n%s", row.tool, row.args, w, raw)
}
}
})
}
}
@@ -19,55 +19,57 @@ const (
// path back: nothing is re-registered, no order reaches the CA, and issuance stops with nothing
// saying why. Naming the cache after the authority means the account is only ever found where it is
// still valid — the new root lands in a directory with no account in it, and autocert registers.
func TestANewCARootMeansANewAccountCache(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot))
if before == after {
t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before)
}
}
func TestTheAccountCacheIsNamedAfterItsAuthority(t *testing.T) {
t.Run("a new CA root means a new account cache", func(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot))
if before == after {
t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before)
}
})
// And the SAME authority keeps the account it registered, restart after restart.
//
// This is the whole reason ACME_CACHE is required in the first place: an account and its
// certificates that did not persist would be re-ordered on every restart, which works silently until
// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the
// mesh writes that file, and a trailing newline coming or going must not throw away an account.
func TestTheSameAuthorityKeepsItsAccount(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n"))
if first != again {
t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s",
first, again)
}
}
// And the SAME authority keeps the account it registered, restart after restart.
//
// This is the whole reason ACME_CACHE is required in the first place: an account and its
// certificates that did not persist would be re-ordered on every restart, which works silently until
// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the
// mesh writes that file, and a trailing newline coming or going must not throw away an account.
t.Run("the same authority keeps its account", func(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n"))
if first != again {
t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s",
first, again)
}
})
// Staging and production are different authorities, and were sharing one account.
//
// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against
// staging reused the staging account, because the cache had no idea they were different.
func TestStagingAndProductionDoNotShareAnAccount(t *testing.T) {
staging := forThisAuthority("/acme", stagingDirectory, nil)
production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil)
if staging == production {
t.Errorf("two issuers share one account: %s", staging)
}
}
// Staging and production are different authorities, and were sharing one account.
//
// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against
// staging reused the staging account, because the cache had no idea they were different.
t.Run("staging and production do not share an account", func(t *testing.T) {
staging := forThisAuthority("/acme", stagingDirectory, nil)
production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil)
if staging == production {
t.Errorf("two issuers share one account: %s", staging)
}
})
// It stays inside the directory the mesh gave it, and is a plain name.
//
// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or
// that carried a separator out of the PEM — would put an account somewhere nothing persists.
func TestTheAccountCacheStaysWhereTheMeshPutIt(t *testing.T) {
const cache = "/var/lib/route-proxy/acme"
got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot))
if !strings.HasPrefix(got, cache+"/") {
t.Fatalf("the account cache is not under %s: %s", cache, got)
}
name := strings.TrimPrefix(got, cache+"/")
if name != filepath.Base(got) || strings.ContainsAny(name, "/.") {
t.Errorf("the account cache is not a plain name: %q", name)
}
// It stays inside the directory the mesh gave it, and is a plain name.
//
// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or
// that carried a separator out of the PEM — would put an account somewhere nothing persists.
t.Run("the account cache stays where the mesh put it", func(t *testing.T) {
const cache = "/var/lib/route-proxy/acme"
got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot))
if !strings.HasPrefix(got, cache+"/") {
t.Fatalf("the account cache is not under %s: %s", cache, got)
}
name := strings.TrimPrefix(got, cache+"/")
if name != filepath.Base(got) || strings.ContainsAny(name, "/.") {
t.Errorf("the account cache is not a plain name: %q", name)
}
})
}
@@ -10,10 +10,23 @@ import (
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/process"
"route-proxy/internal/routestate"
)
// keptStates is every state route-proxy keeps, as both its halves write them: the routes and the refusals this
// proxy writes, readable by others, and the bound entries the provider helper writes.
func keptStates() []announce.State {
return []announce.State{
{Name: routestate.BoundState},
{Name: routestate.RoutesState, Readable: true},
{Name: routestate.UnroutedState, Readable: true},
}
}
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
//
// **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
@@ -44,6 +57,14 @@ func followOnce(path string, held *table, fromBus *atomic.Bool) error {
if err != nil {
return fmt.Errorf("connecting to the bus at %s: %w", membership.BareAddress(account.URL), err)
}
// What this proxy keeps, said on the bus (novox/hq ADR 0495; issue 774): the names it serves and the names it
// refused, readable by any module granted a read of them; the provider helper's entries, read by the machines'
// clients through their own grant. Said before the first write, so no write of an unannounced state is refused.
if err := conn.Keeps(keptStates()...); err != nil {
log.Printf("cannot say on the bus what this proxy keeps, so it keeps nothing there: %v", err)
} else {
held.ledger.attach(account.Node, conn.State(routestate.RoutesState), conn.State(routestate.UnroutedState))
}
subject := membership.Subject(account.Node, account.Module)
err = conn.FollowMembership(func(issued membership.Membership) {
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
@@ -0,0 +1,552 @@
package main
import (
"context"
"crypto/x509"
"encoding/json"
"encoding/pem"
"log"
"net/http"
"reflect"
"sort"
"strings"
"sync"
"time"
"golang.org/x/crypto/acme/autocert"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// What this proxy says on the bus about the names it serves and the names it refuses (issue 774).
//
// **On change, never on a timer.** A route arriving or leaving, a certificate issued or failing, an upstream answer
// that differs from the last one written, a name asked for that is not served: each marks its key, and one writer
// puts the marked keys. An answer that is the same as the last one written is written again only once it is older
// than refreshAfter, so a site answering 200 all day costs one write a minute, not one per request.
//
// Until the bus is reached nothing is written; what happened meanwhile is held and written when it is.
// refreshAfter is how old a written answer, refusal or certificate failure may be before the same one is written
// again with its new time.
const refreshAfter = time.Minute
// mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes.
const mostAsked = 4096
// newPerMinute is how many names not seen before are kept by name in a minute; past it they are only counted.
const newPerMinute = 5
// kvStore is one of the module's states, as process.State is one.
type kvStore interface {
Put(key string, value any) (uint64, error)
Delete(key string) error
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// ledger is what this proxy keeps of what it serves and refuses, and writes to the bus.
type ledger struct {
mu sync.Mutex
now func() time.Time
node string
routes, unrouted kvStore
served map[string]*routestate.Route
previous map[string]routestate.Route // what the bus held for this machine when it was reached
asked map[string]*routestate.Unrouted
answerWritten map[string]time.Time
askedWritten map[string]time.Time
dirtyRoutes map[string]bool // a name to put when served, to delete when not
dirtyAsked map[string]bool
// domains are the registrable domains of the names served: a refused name under one is kept by name.
domains map[string]bool
newMinute time.Time
newInMinute int
other routestate.Other
otherMinute time.Time
otherInMinute int
otherDirty bool
routesFailing, askFailed bool // a failed write said once, until one succeeds
kick chan struct{}
}
func newLedger() *ledger {
return &ledger{
now: time.Now,
served: map[string]*routestate.Route{},
previous: map[string]routestate.Route{},
asked: map[string]*routestate.Unrouted{},
answerWritten: map[string]time.Time{},
askedWritten: map[string]time.Time{},
dirtyRoutes: map[string]bool{},
dirtyAsked: map[string]bool{},
domains: map[string]bool{},
kick: make(chan struct{}, 1),
}
}
// loop writes whatever was marked, each time something was.
func (l *ledger) loop() {
for range l.kick {
l.flush()
}
}
func (l *ledger) wake() {
select {
case l.kick <- struct{}{}:
default:
}
}
// attach is the bus reached, as node: what the bus held for this machine is read back, so a restart keeps a name's
// certificate and last answer, and everything held is written.
func (l *ledger) attach(node string, routes, unrouted kvStore) {
if l == nil {
return
}
before := map[string]routestate.Route{}
if keys, err := routes.Keys(); err != nil {
log.Printf("cannot read back what this proxy said it serves: %v", err)
} else {
for _, key := range keys {
at, name := routestate.Split(key)
if at != node {
continue
}
entry, err := routes.Get(key)
if err != nil || entry == nil {
continue
}
var r routestate.Route
if json.Unmarshal(entry.Value, &r) == nil {
before[name] = r
}
}
}
l.mu.Lock()
l.node, l.routes, l.unrouted = node, routes, unrouted
for name, r := range before {
l.previous[name] = r
l.dirtyRoutes[name] = true // put again when served, deleted when no longer
if now := l.served[name]; now != nil {
keepWhatWasKnown(now, r)
}
}
for name := range l.served {
l.dirtyRoutes[name] = true
}
for name := range l.asked {
l.dirtyAsked[name] = true
}
l.mu.Unlock()
l.wake()
}
// keepWhatWasKnown carries a certificate and a last answer read back from the bus onto a route that knows none yet.
func keepWhatWasKnown(r *routestate.Route, before routestate.Route) {
if r.Certificate.Outcome == "" && r.Certificate.NotAfter == nil && before.Certificate.Authority == r.Certificate.Authority {
r.Certificate = before.Certificate
}
if r.LastAnswer == nil {
r.LastAnswer = before.LastAnswer
}
}
// serving is the table this proxy now serves: a name new or changed is written, a name gone is deleted, and a name
// that was asked for unserved and now is served leaves the unrouted state.
func (l *ledger) serving(routes map[string][]rule, public map[string]bool) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
for name, rules := range routes {
reach := routestate.ReachInternal
if public[name] {
reach = routestate.ReachPublic
}
targets := targetsOf(rules)
r := l.served[name]
if r == nil {
r = &routestate.Route{Name: name}
if before, ok := l.previous[name]; ok {
r.Certificate.Authority = reach
keepWhatWasKnown(r, before)
}
l.served[name] = r
l.dirtyRoutes[name] = true
}
if r.Reach != reach || !reflect.DeepEqual(r.Targets, targets) {
r.Reach, r.Targets = reach, targets
l.dirtyRoutes[name] = true
}
if r.Certificate.Authority != reach {
r.Certificate = routestate.Certificate{Authority: reach}
l.dirtyRoutes[name] = true
}
if _, was := l.asked[name]; was {
delete(l.asked, name)
l.dirtyAsked[name] = true
}
}
l.domains = map[string]bool{}
for name := range routes {
if d := routestate.Registrable(name); d != "" {
l.domains[d] = true
}
}
for name := range l.served {
if _, still := routes[name]; !still {
delete(l.served, name)
delete(l.answerWritten, name)
l.dirtyRoutes[name] = true
}
}
l.wake()
}
func targetsOf(rules []rule) []routestate.Target {
out := make([]routestate.Target, 0, len(rules))
for _, r := range rules {
out = append(out, routestate.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target,
Deny: r.policy.deny, Redirect: r.policy.redirectTo, Auth: len(r.policy.users) > 0 || r.policy.sealed != "",
Sealed: r.policy.sealed})
}
return out
}
// answered is an upstream answer to a request for name: its status, or the error that stood in for one.
func (l *ledger) answered(name, path string, status int, failed error) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
why := ""
if failed != nil {
why = failed.Error()
}
before := r.LastAnswer
r.LastAnswer = &routestate.Answer{Status: status, At: now, Error: why, Path: path}
if before != nil && before.Status == status && before.Error == why && now.Sub(l.answerWritten[name]) < refreshAfter {
return
}
l.answerWritten[name] = now
l.dirtyRoutes[name] = true
l.wake()
}
// refused is a name this proxy answered "no route for": in a request, or in a handshake's server name.
//
// **Only a name under a domain this proxy serves is kept by name**, and at most newPerMinute new ones a minute: a scan
// sends any name it likes, and each kept name is a key on the bus. Every other refusal is counted, and the count is
// written once a minute under the proxy's OtherName key. Nothing here scans: forgetting old names is the writer's.
// Who asked is not kept: the journal line beside it is what the machine's intrusion prevention reads.
func (l *ledger) refused(name string, internalOnly, handshake bool) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
if _, isServed := l.served[name]; isServed && !internalOnly {
return
}
now := l.now()
a := l.asked[name]
if a == nil {
if !routestate.Keyable(name) || !l.domains[routestate.Registrable(name)] || !l.mayAddName(now) {
l.countOther(now)
return
}
a = &routestate.Unrouted{Name: name}
l.asked[name] = a
}
a.AskedAt, a.InternalOnly, a.Handshake = now, internalOnly, handshake
a.Times++
if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter {
return
}
l.askedWritten[name] = now
l.dirtyAsked[name] = true
l.wake()
}
// mayAddName says whether another new name may be kept this minute, and counts it.
func (l *ledger) mayAddName(now time.Time) bool {
if minute := now.Truncate(time.Minute); !minute.Equal(l.newMinute) {
l.newMinute, l.newInMinute = minute, 0
}
if l.newInMinute >= newPerMinute {
return false
}
l.newInMinute++
return true
}
// countOther counts a refusal not kept by name. A minute's count is written when the next minute's first one comes.
func (l *ledger) countOther(now time.Time) {
minute := now.Truncate(time.Minute)
if !minute.Equal(l.otherMinute) {
if l.otherInMinute > 0 {
l.other.Minute, l.other.LastMinute = l.otherMinute, l.otherInMinute
l.otherDirty = true
l.wake()
}
l.otherMinute, l.otherInMinute = minute, 0
}
l.otherInMinute++
l.other.SinceStart++
}
// forgetOldestAsked drops the names asked longest ago from memory until mostAsked are left; the bus keeps them until
// their time to live ends. Called by the writer, never on a request.
func (l *ledger) forgetOldestAsked() {
if len(l.asked) <= mostAsked {
return
}
names := make([]string, 0, len(l.asked))
for name := range l.asked {
names = append(names, name)
}
sort.Slice(names, func(i, j int) bool { return l.asked[names[i]].AskedAt.Before(l.asked[names[j]].AskedAt) })
for _, name := range names[:len(names)-mostAsked] {
delete(l.asked, name)
delete(l.askedWritten, name)
delete(l.dirtyAsked, name)
}
}
// certified is a certificate an authority issued for name, as autocert stored it.
func (l *ledger) certified(name string, leaf *x509.Certificate) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
notAfter := leaf.NotAfter.UTC()
r.Certificate.Issuer, r.Certificate.NotAfter = issuerOf(leaf), &notAfter
r.Certificate.RenewedAt, r.Certificate.At = &now, &now
r.Certificate.Outcome, r.Certificate.Error = routestate.Issued, ""
l.dirtyRoutes[name] = true
l.wake()
}
// handedOut is the certificate a handshake for name was answered with: written when it is not the one recorded.
func (l *ledger) handedOut(name string, leaf *x509.Certificate) {
if l == nil || leaf == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
notAfter := leaf.NotAfter.UTC()
if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != routestate.Failed {
return
}
now := l.now()
r.Certificate.Issuer, r.Certificate.NotAfter, r.Certificate.At = issuerOf(leaf), &notAfter, &now
if r.Certificate.Outcome != routestate.Issued || r.Certificate.Error != "" {
r.Certificate.Outcome, r.Certificate.Error = routestate.Cached, ""
}
l.dirtyRoutes[name] = true
l.wake()
}
// certificateFailed is a handshake for name that found no certificate to answer with, and why.
func (l *ledger) certificateFailed(name string, failed error) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
same := r.Certificate.Outcome == routestate.Failed && r.Certificate.Error == failed.Error()
if same && r.Certificate.At != nil && now.Sub(*r.Certificate.At) < refreshAfter {
return
}
r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = routestate.Failed, failed.Error(), &now
l.dirtyRoutes[name] = true
l.wake()
}
func issuerOf(leaf *x509.Certificate) string {
if cn := leaf.Issuer.CommonName; cn != "" {
if len(leaf.Issuer.Organization) > 0 {
return leaf.Issuer.Organization[0] + " " + cn
}
return cn
}
return leaf.Issuer.String()
}
// flush writes every marked key. A route's write that fails stays marked and is tried again at the next change; a
// refused name's write that fails is dropped, never tried again: the bucket refusing it is full, or the bus is away,
// and either way it is not worth a storm of retries. Each kind of failure is logged once, until a write succeeds.
func (l *ledger) flush() {
type put struct {
store kvStore
key string
value any // nil deletes
route bool
}
l.mu.Lock()
if l.routes == nil {
l.mu.Unlock()
return
}
l.forgetOldestAsked()
var puts []put
for name := range l.dirtyRoutes {
p := put{store: l.routes, key: routestate.Key(l.node, name), route: true}
if r := l.served[name]; r != nil {
c := *r
c.Proxy = l.node
p.value = c
}
puts = append(puts, p)
}
for name := range l.dirtyAsked {
p := put{store: l.unrouted, key: routestate.Key(l.node, name)}
if a := l.asked[name]; a != nil {
c := *a
c.Proxy = l.node
p.value = c
}
puts = append(puts, p)
}
if l.otherDirty {
o := l.other
o.Proxy = l.node
puts = append(puts, put{store: l.unrouted, key: routestate.Key(l.node, routestate.OtherName), value: o})
}
l.dirtyRoutes, l.dirtyAsked, l.otherDirty = map[string]bool{}, map[string]bool{}, false
l.mu.Unlock()
for _, p := range puts {
var err error
if p.value == nil {
err = p.store.Delete(p.key)
} else {
_, err = p.store.Put(p.key, p.value)
}
l.mu.Lock()
switch {
case err == nil && p.route:
l.routesFailing = false
case err == nil:
l.askFailed = false
case p.route:
if !l.routesFailing {
log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err)
}
l.routesFailing = true
_, name := routestate.Split(p.key)
l.dirtyRoutes[name] = true
default:
if !l.askFailed {
log.Printf("cannot write %s to the bus, dropped with every refused name until a write succeeds: %v",
p.key, err)
}
l.askFailed = true
}
l.mu.Unlock()
}
}
// recordingCache is an authority's certificate cache that tells the ledger of every certificate stored in it: the
// one place autocert says it was issued one.
type recordingCache struct {
autocert.Cache
held *table
}
func (c recordingCache) Put(ctx context.Context, key string, data []byte) error {
if err := c.Cache.Put(ctx, key, data); err != nil {
return err
}
name := strings.TrimSuffix(key, "+rsa")
if strings.Contains(name, "+") || !c.held.routed(name) {
return nil // the account key, a challenge token
}
if leaf := leafOf(data); leaf != nil {
c.held.ledger.certified(bareHost(name), leaf)
}
return nil
}
// leafOf is the first certificate in what autocert stores: the private key, then the chain, in PEM.
func leafOf(data []byte) *x509.Certificate {
for {
var block *pem.Block
block, data = pem.Decode(data)
if block == nil {
return nil
}
if block.Type == "CERTIFICATE" {
leaf, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil
}
return leaf
}
}
}
// answerRecorder carries a proxied request's answer through and remembers its status and the proxy's error.
type answerRecorder struct {
http.ResponseWriter
status int
err error
}
func (a *answerRecorder) WriteHeader(status int) {
if a.status == 0 {
a.status = status
}
a.ResponseWriter.WriteHeader(status)
}
func (a *answerRecorder) Write(b []byte) (int, error) {
if a.status == 0 {
a.status = http.StatusOK
}
return a.ResponseWriter.Write(b)
}
// Unwrap lets http.ResponseController reach the connection beneath: a flush, an upgrade's hijack.
func (a *answerRecorder) Unwrap() http.ResponseWriter { return a.ResponseWriter }
// proxyFailed is the reverse proxy's answer when upstream gave none: the error kept for the ledger, and a 502.
func proxyFailed(w http.ResponseWriter, r *http.Request, err error) {
if a, ok := w.(*answerRecorder); ok {
a.err = err
}
log.Printf("http: proxy error for %s%s: %v", r.Host, r.URL.Path, err)
w.WriteHeader(http.StatusBadGateway)
}
@@ -0,0 +1,291 @@
package main
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"net/http"
"net/http/httptest"
"sort"
"strings"
"testing"
"time"
"golang.org/x/crypto/acme/autocert"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// fakeState is a state on the bus, in memory, counting what was written to it.
type fakeState struct {
values map[string]json.RawMessage
puts int
tries int // every Put asked, refused or not
fail error // what every Put is refused with, while set
}
func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} }
func (f *fakeState) Put(key string, value any) (uint64, error) {
f.tries++
if f.fail != nil {
return 0, f.fail
}
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
f.values[key] = raw
f.puts++
return uint64(f.puts), nil
}
func (f *fakeState) Delete(key string) error { delete(f.values, key); return nil }
func (f *fakeState) Keys() ([]string, error) {
out := []string{}
for k := range f.values {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (f *fakeState) Get(key string) (*stdio.StateEntry, error) {
v, ok := f.values[key]
if !ok {
return nil, nil
}
return &stdio.StateEntry{Key: key, Value: v}, nil
}
func (f *fakeState) route(t *testing.T, key string) *routestate.Route {
t.Helper()
raw, ok := f.values[key]
if !ok {
return nil
}
var r routestate.Route
if err := json.Unmarshal(raw, &r); err != nil {
t.Fatal(err)
}
return &r
}
// issuedPEM is what autocert stores for a name: the key, then the certificate, signed by an issuer named here.
func issuedPEM(t *testing.T, name string, notAfter time.Time) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{SerialNumber: big.NewInt(1), DNSNames: []string{name},
Subject: pkix.Name{CommonName: name}, Issuer: pkix.Name{CommonName: "R11", Organization: []string{"Let's Encrypt"}},
NotBefore: notAfter.Add(-90 * 24 * time.Hour), NotAfter: notAfter}
issuer := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: template.Issuer, NotBefore: template.NotBefore,
NotAfter: notAfter, IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
keyDER, _ := x509.MarshalECPrivateKey(key)
out := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
return append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
}
// What the proxy tells the bus (issue 774): each served name with where it goes, its certificate and the last
// upstream answer; each name refused; and only on change. Without the ledger nothing reaches the bus, and the
// operator's question "is grafana reachable" is answered with curl.
func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) {
notAfter := time.Date(2027, 1, 28, 12, 0, 0, 0, time.UTC)
gone := func() string { // an address nothing listens on: the upstream that does not answer
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
return l.Addr().String()
}()
_, port, _ := net.SplitHostPort(gone)
contributions := []contribution{
{From: "grafana", Node: "anchor", At: "127.0.0.1", Values: map[string]any{"name": "grafana.example", "port": mustFloat(t, port)}},
{From: "drive", Node: "home-server", At: "10.0.0.2", Values: map[string]any{"internal-name": "drive.internal", "port": 8080.0}},
}
for _, row := range []struct {
name string
act func(t *testing.T, held *table)
check func(t *testing.T, routes, unrouted *fakeState)
}{
{"a served name is written with the module and node it forwards to",
func(t *testing.T, held *table) {},
func(t *testing.T, routes, _ *fakeState) {
r := routes.route(t, "anchor.grafana.example")
if r == nil || r.Proxy != "anchor" || r.Reach != routestate.ReachPublic || len(r.Targets) != 1 ||
r.Targets[0].Module != "grafana" || r.Targets[0].Node != "anchor" || r.Targets[0].Target != "http://127.0.0.1:"+port {
t.Fatalf("grafana's route is not on the bus as served: %+v", r)
}
if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != routestate.ReachInternal ||
d.Certificate.Authority != routestate.ReachInternal {
t.Fatalf("an internal-only name is not on the bus as internal: %+v", d)
}
}},
{"a withdrawn route is deleted",
func(t *testing.T, held *table) {
routes, public := routesOf(contributions[1:])
held.set(routes, public)
},
func(t *testing.T, routes, _ *fakeState) {
if routes.route(t, "anchor.grafana.example") != nil {
t.Fatal("a route no longer served is still on the bus")
}
}},
{"an upstream that does not answer is written as a 502 with why, once a minute",
func(t *testing.T, held *table) {
for range 3 {
req := httptest.NewRequest(http.MethodGet, "http://grafana.example/login", nil)
handler(held).ServeHTTP(httptest.NewRecorder(), req)
}
},
func(t *testing.T, routes, _ *fakeState) {
r := routes.route(t, "anchor.grafana.example")
if r == nil || r.LastAnswer == nil || r.LastAnswer.Status != http.StatusBadGateway ||
r.LastAnswer.Error == "" || r.LastAnswer.Path != "/login" {
t.Fatalf("the failed answer is not on the bus: %+v", r)
}
if routes.puts != 3 { // the two routes, then the first answer; the same answer again is not written
t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts)
}
}},
{"a name under a served domain is written as asked, without who asked, and leaves when it is served",
func(t *testing.T, held *table) {
req := httptest.NewRequest(http.MethodGet, "http://wiki.grafana.example/", nil)
req.RemoteAddr = "192.0.2.7:51000"
handler(held).ServeHTTP(httptest.NewRecorder(), req)
held.ledger.flush()
var a routestate.Unrouted
raw := held.ledger.unrouted.(*fakeState).values["anchor.wiki.grafana.example"]
if err := json.Unmarshal(raw, &a); err != nil || a.Times != 1 || a.AskedAt.IsZero() {
t.Fatalf("the refused name is not on the bus as asked: %s", raw)
}
if strings.Contains(string(raw), "192.0.2.7") || strings.Contains(string(raw), `"from"`) {
t.Fatalf("who asked is on the bus: %s", raw)
}
routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor",
Values: map[string]any{"name": "wiki.grafana.example", "port": 3000.0}}))
held.set(routes, public)
},
func(t *testing.T, _, unrouted *fakeState) {
if len(unrouted.values) != 0 {
t.Fatalf("a name now served is still written as unrouted: %v", unrouted.values)
}
}},
{"a handshake for a name not routed is written as asked",
func(t *testing.T, held *table) {
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return nil, errors.New("no public route for \"forgotten.grafana.example\" in this mesh, so no certificate is asked for")
}, nil)
_, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.grafana.example"})
},
func(t *testing.T, _, unrouted *fakeState) {
var a routestate.Unrouted
if err := json.Unmarshal(unrouted.values["anchor.forgotten.grafana.example"], &a); err != nil || !a.Handshake {
t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values)
}
}},
{"a flood of names writes at most the limit by name, counts the rest once a minute, and never retries a refused write",
func(t *testing.T, held *table) {
clock := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
held.ledger.now = func() time.Time { return clock }
full := held.ledger.unrouted.(*fakeState)
for i := range 40 {
held.ledger.refused(fmt.Sprintf("x%d.grafana.example", i), false, false) // under a served domain
held.ledger.refused(fmt.Sprintf("x%d.scan.test", i), false, true) // under none
}
held.ledger.flush()
if named := len(full.values); named != newPerMinute {
t.Fatalf("a flood wrote %d names, want the limit %d: %v", named, newPerMinute, full.values)
}
full.fail = errors.New("nats: maximum bytes exceeded")
clock = clock.Add(time.Minute)
held.ledger.refused("y.grafana.example", false, false) // a new minute: one more name,
held.ledger.refused("v.scan.test", false, false) // and the last minute's count
held.ledger.flush()
tries := full.tries
held.ledger.refused("y.grafana.example", false, false)
held.ledger.refused("z.scan.test", false, false)
held.ledger.flush()
if full.tries != tries {
t.Fatalf("writes the bucket refused were tried again: %d tries, then %d", tries, full.tries)
}
full.fail = nil
clock = clock.Add(time.Minute)
held.ledger.refused("w.scan.test", false, false)
},
func(t *testing.T, _, unrouted *fakeState) {
var o routestate.Other
if err := json.Unmarshal(unrouted.values["anchor._other"], &o); err != nil || o.LastMinute != 2 || o.SinceStart != 78 {
t.Fatalf("the refusals not kept by name are not counted on the bus: %s", unrouted.values["anchor._other"])
}
}},
{"a certificate an authority issued is written with its issuer, expiry and renewal",
func(t *testing.T, held *table) {
cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held}
if err := cache.Put(context.Background(), "grafana.example", issuedPEM(t, "grafana.example", notAfter)); err != nil {
t.Fatal(err)
}
_ = cache.Put(context.Background(), "acme_account+key", []byte("not a certificate"))
},
func(t *testing.T, routes, _ *fakeState) {
c := routes.route(t, "anchor.grafana.example").Certificate
if c.Outcome != routestate.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil ||
!c.NotAfter.Equal(notAfter) || c.RenewedAt == nil {
t.Fatalf("the issued certificate is not on the bus: %+v", c)
}
}},
{"a certificate that cannot be had is written with why",
func(t *testing.T, held *table) {
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return nil, errors.New("acme: urn:ietf:params:acme:error:rateLimited")
}, nil)
_, _ = pick(&tls.ClientHelloInfo{ServerName: "grafana.example"})
},
func(t *testing.T, routes, _ *fakeState) {
c := routes.route(t, "anchor.grafana.example").Certificate
if c.Outcome != routestate.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" {
t.Fatalf("the failed certificate is not on the bus: %+v", c)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
held := newTable()
held.ledger = newLedger()
routes, unrouted := newFakeState(), newFakeState()
held.ledger.attach("anchor", routes, unrouted)
table, public := routesOf(contributions)
held.set(table, public)
held.ledger.flush()
row.act(t, held)
held.ledger.flush()
row.check(t, routes, unrouted)
})
}
}
func mustFloat(t *testing.T, port string) float64 {
t.Helper()
var n float64
if err := json.Unmarshal([]byte(port), &n); err != nil {
t.Fatal(err)
}
return n
}
+48 -7
View File
@@ -179,6 +179,8 @@ type rule struct {
// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
// a proxy's own default refuses them long before the workload is reached.
maxRequestBody int64
// from and node are the consumer that contributed the rule and its machine, which the bus is told (issue 774).
from, node string
}
// table is what the proxy is currently serving, replaced whole whenever the file changes.
@@ -202,6 +204,8 @@ type table struct {
// machine's address on the private network, as the mesh issued it in this proxy's membership
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
inside sources
// ledger is what this proxy says on the bus about what it serves and refuses (issue 774); nil says nothing.
ledger *ledger
}
// sources is who may be served an internal name: the private network's addresses as the mesh
@@ -289,6 +293,7 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) {
t.to = made
t.public = public
t.mu.Unlock()
t.ledger.serving(made, public)
}
// inOrder puts the rules for one host into the order they are matched in, and does so totally.
@@ -437,6 +442,8 @@ func run() error {
}
held := newTable()
held.ledger = newLedger()
go held.ledger.loop()
// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
// it an internal name is served to this machine and to nobody else — refused, never opened.
@@ -487,7 +494,7 @@ func run() error {
"to persist, or every restart orders them again")
}
publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
onlyWhatTheMeshSaid(held))
onlyWhatTheMeshSaid(held), held)
if err != nil {
return err
}
@@ -499,7 +506,7 @@ func run() error {
var internalManager *autocert.Manager
if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
onlyInternalNamesTheMeshSaid(held))
onlyInternalNamesTheMeshSaid(held), held)
if err != nil {
return fmt.Errorf("internal certificate authority: %w", err)
}
@@ -619,7 +626,7 @@ func (p *probedResponse) replayTo(w http.ResponseWriter) {
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
func newManager(cache, directory, bundle string, policy autocert.HostPolicy, held *table) (*autocert.Manager, error) {
client := &acme.Client{DirectoryURL: directory}
var root []byte
if bundle != "" {
@@ -651,7 +658,7 @@ func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*a
// different names because their directories differ.
mine := forThisAuthority(cache, directory, root)
return &autocert.Manager{
Cache: autocert.DirCache(mine),
Cache: recordingCache{Cache: autocert.DirCache(mine), held: held},
Prompt: autocert.AcceptTOS,
HostPolicy: policy,
Client: client,
@@ -700,9 +707,10 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
if internalManager != nil {
fromInternal = internalManager.TLSConfig().GetCertificate
}
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
held.ledger.refused(bareHost(hello.ServerName), true, true)
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
hello.ServerName)
}
@@ -712,6 +720,28 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
}
return fromPublic(hello)
}
// What the bus is told of a handshake (issue 774): a routed name's certificate, or why it has none; a name not
// routed here, which a browser asking for a site the mesh forgot to route meets before any request is made.
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
cert, err := pick(hello)
name := bareHost(hello.ServerName)
switch {
case name == "":
case !held.routed(name):
held.ledger.refused(name, false, true)
case err != nil:
if !strings.HasPrefix(err.Error(), "no public route for") {
held.ledger.certificateFailed(name, err)
}
case cert != nil:
leaf := cert.Leaf
if leaf == nil && len(cert.Certificate) > 0 {
leaf, _ = x509.ParseCertificate(cert.Certificate[0])
}
held.ledger.handedOut(name, leaf)
}
return cert, err
}
}
// newTable is an empty routing table.
@@ -737,6 +767,9 @@ func handler(held *table) http.Handler {
// this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the
// jail's filter expects it.
log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
if hidden || !held.routed(r.Host) {
held.ledger.refused(bareHost(r.Host), hidden, false)
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound)
if !hidden && held.routed(r.Host) {
@@ -786,7 +819,14 @@ func handler(held *table) http.Handler {
r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
}
matched.to.ServeHTTP(w, r)
// The upstream's answer is what the bus is told of this name (issue 774): its status, or why there was none.
answer := &answerRecorder{ResponseWriter: w}
matched.to.ServeHTTP(answer, r)
status := answer.status
if status == 0 && answer.err == nil && r.Header.Get("Upgrade") != "" {
status = http.StatusSwitchingProtocols
}
held.ledger.answered(bareHost(r.Host), r.URL.Path, status, answer.err)
})
}
@@ -955,6 +995,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
}
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
}
made.from, made.node = c.From, c.Node
if name != "" {
host := strings.ToLower(name)
@@ -1072,5 +1113,5 @@ func towards(where *url.URL) *httputil.ReverseProxy {
pr.SetURL(where)
pr.Out.Host = pr.In.Host
pr.SetXForwarded()
}}
}, ErrorHandler: proxyFailed}
}
@@ -121,19 +121,6 @@ func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
}
}
// A route with neither name has nothing to be served under, and is still skipped.
func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 || len(public) != 0 {
t.Errorf("a route that named nothing was served: %v %v", routes, public)
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
@@ -161,21 +148,6 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
}
}
// Skipped rather than served wrongly. A route with no port would proxy to :0.
func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 1 || targetOf(routes, "fine.example") == "" {
t.Fatalf("an unusable contribution was served: %v", routes)
}
}
// A route may name a target reached over https, for a backend that terminates its own TLS — the
// shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise.
func TestARouteMayTargetHttps(t *testing.T) {
@@ -194,19 +166,6 @@ func TestARouteMayTargetHttps(t *testing.T) {
}
}
// A scheme that is neither http nor https is refused rather than guessed at.
func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 {
t.Fatalf("a route with an unusable scheme was served: %v", routes)
}
}
// End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still
// reached when the route declared `insecure`, and the response comes back through unmodified.
func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) {
@@ -455,23 +414,6 @@ func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
}
}
// A limit that is not a whole positive number of bytes takes the route with it. Serving it without
// the limit would carry exactly what the module said not to carry, and report success doing it.
func TestARouteWithAnUnusableLimitIsSkipped(t *testing.T) {
for _, asked := range []string{`"lots"`, `-1`, `0`, `1.5`} {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"anchor.internal",
"values":{"name":"images.example","port":5000,"max-request-body":`+asked+`}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes["images.example"]) != 0 {
t.Errorf("a route asking for a max-request-body of %s was served anyway: %v", asked, routes)
}
}
}
// A request larger than the route carries is refused by the proxy, with the limit named, and the
// workload never sees it. A request within it is proxied normally.
func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
@@ -532,3 +474,32 @@ func post(url, host, body string) (*http.Response, error) {
asked.Header.Set("Content-Type", "application/octet-stream")
return http.DefaultClient.Do(asked)
}
// A contribution that cannot be served is skipped rather than served wrongly: one that names nothing to be served
// under; one with no port, which would proxy to :0; a scheme that is neither http nor https, refused rather than
// guessed at; and a body limit that is not a whole positive number of bytes, since serving it without the limit would
// carry exactly what the module said not to carry, and report success doing it.
func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) {
for _, row := range []struct{ why, given, served string }{
{"neither name", `{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}`, ""},
{"no port", `{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}`, "fine.example"},
{"an unknown scheme", `{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}`, ""},
{"a limit in words", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":"lots"}}`, ""},
{"a negative limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":-1}}`, ""},
{"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""},
{"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""},
} {
routes, public, err := routesFrom(write(t, `{"given":[`+row.given+`]}`))
if err != nil {
t.Fatal(err)
}
switch {
case row.served == "" && (len(routes) != 0 || len(public) != 0):
t.Errorf("%s: an unusable contribution was served: %v %v", row.why, routes, public)
case row.served != "" && (len(routes) != 1 || targetOf(routes, row.served) == ""):
t.Errorf("%s: an unusable contribution was served beside the usable one: %v", row.why, routes)
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module route-proxy
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68 h1:2rMJX0RMIBaN5NDIaEZVwbj5MFism1O3bRayEz+WlOs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -0,0 +1,151 @@
// Package routestate is what the web front keeps on the bus about the names it serves, and about the names it was asked
// for and does not serve (issue 774): the shape both halves of route-proxy agree on. The proxy writes it, on change,
// in its states `routes` and `unrouted`, announced readable; the module's tools read it from there, so the question
// "is this site reachable, and is its certificate good" is answered from the bus rather than with curl.
//
// A key is `<node>.<name>`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds
// no dot, so the first dot splits them.
package routestate
import (
"net"
"regexp"
"strings"
"time"
)
// The states route-proxy keeps, by their local names (the buckets are route-proxy_<name>).
const (
// RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got.
RoutesState = "routes"
// UnroutedState holds one key per name the proxy answered "no route for" with, and when it was last asked.
UnroutedState = "unrouted"
// BoundState is the provider helper's bucket of consumer entries (mesh-sdk go/provider).
BoundState = "bound"
)
// The words a route's reach and a certificate's authority are said in.
const (
ReachPublic = "public"
ReachInternal = "internal"
)
// The words a certificate's outcome is said in.
const (
// Issued: the authority issued it while this proxy ran, at RenewedAt.
Issued = "issued"
// Cached: this proxy served it from what it kept, and saw no issuance of it since it started.
Cached = "cached"
// Failed: the last attempt to have one failed, Error says why.
Failed = "failed"
)
// Target is one rule a name is served by: what a request matching it reaches, or what it is answered with.
type Target struct {
// Path is the path prefix the rule is scoped to; empty is every path.
Path string `json:"path,omitempty"`
// Module and Node are the consumer that contributed the route and the machine it runs on.
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
// Target is where the request is forwarded, scheme://address:port; empty for a rule that answers itself.
Target string `json:"target,omitempty"`
Deny bool `json:"deny,omitempty"`
Redirect string `json:"redirect,omitempty"`
// Auth says the rule asks for credentials; Sealed says why it cannot check them, and so refuses everything.
Auth bool `json:"auth,omitempty"`
Sealed string `json:"sealed,omitempty"`
}
// Certificate is what this proxy knows of a name's certificate.
type Certificate struct {
// Authority is which issuer certifies the name: public (the ACME directory the module names) or internal (the
// mesh's own).
Authority string `json:"authority"`
Issuer string `json:"issuer,omitempty"`
NotAfter *time.Time `json:"not_after,omitempty"`
// RenewedAt is when an authority last issued it to this proxy.
RenewedAt *time.Time `json:"renewed_at,omitempty"`
// Outcome is issued, cached or failed; empty until a handshake or an issuance said anything.
Outcome string `json:"outcome,omitempty"`
Error string `json:"error,omitempty"`
At *time.Time `json:"at,omitempty"`
}
// Answer is the last answer the proxy got from upstream for a name: the status it carried back, or the error that
// stood in for one (a 502).
type Answer struct {
Status int `json:"status"`
At time.Time `json:"at"`
Error string `json:"error,omitempty"`
Path string `json:"path,omitempty"`
}
// Route is one served name, the value of a key of RoutesState.
type Route struct {
Name string `json:"name"`
// Proxy is the machine whose proxy serves it.
Proxy string `json:"proxy"`
// Reach is public (a route's own name) or internal (only its internal-name, served inside the private network).
Reach string `json:"reach"`
Targets []Target `json:"targets"`
Certificate Certificate `json:"certificate"`
LastAnswer *Answer `json:"last_answer,omitempty"`
}
// Unrouted is one name the proxy answered "no route for", the value of a key of UnroutedState.
type Unrouted struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
// AskedAt is when the name was last asked for. Who asked is not kept here: the journal keeps it, for the
// machine's intrusion prevention.
AskedAt time.Time `json:"asked_at"`
// Times is how often it was asked since the proxy last started.
Times int `json:"times"`
// InternalOnly says the name is served, but only inside the private network, and was asked from outside.
InternalOnly bool `json:"internal_only,omitempty"`
// Handshake says it was asked in a TLS handshake (the name a client sent), not in an HTTP request.
Handshake bool `json:"handshake,omitempty"`
}
// OtherName is the name of the one key per proxy that counts the refusals not kept by name: names under no domain
// the proxy serves (what a scan sends), and names past the limit of new names a minute. Its key is
// `<node>._other`; no host name begins with an underscore.
const OtherName = "_other"
// Other is the value of a proxy's OtherName key.
type Other struct {
Proxy string `json:"proxy"`
// Minute is the last whole minute counted, and LastMinute how many refusals it held.
Minute time.Time `json:"minute"`
LastMinute int `json:"last_minute"`
// SinceStart is every refusal not kept by name since the proxy started.
SinceStart int `json:"since_start"`
}
// Registrable is the domain a name is registered under, as its last two labels; "" for an address or a single
// label. Two labels is wrong for a public suffix of two (co.uk); it only decides which refused names are kept by name.
func Registrable(name string) string {
if net.ParseIP(strings.Trim(name, "[]")) != nil {
return ""
}
labels := strings.Split(name, ".")
if len(labels) < 2 {
return ""
}
return strings.Join(labels[len(labels)-2:], ".")
}
var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`)
// Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request
// is whatever a client sent; one no key could hold is no site anybody runs.
func Keyable(name string) bool { return keyable.MatchString(name) && !strings.Contains(name, "..") }
// Key is the key of name served or asked at node's proxy.
func Key(node, name string) string { return node + "." + name }
// Split is a key's node and name.
func Split(key string) (node, name string) {
node, name, _ = strings.Cut(key, ".")
return node, name
}
+24 -2
View File
@@ -244,9 +244,31 @@
}
],
"state": [
"bound"
"bound",
"routes",
{
"name": "unrouted",
"ttl-seconds": 604800
}
],
"tools": [
"route_proxy_routes",
"route_proxy_route",
"route_proxy_unrouted"
],
"replaces": {
"route_proxy_routes": [
"curl -I"
],
"route_proxy_route": [
"openssl s_client -connect"
],
"route_proxy_unrouted": [
"docker logs route-proxy"
]
},
"reads": [
"mesh-controller.assignments"
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
]
}
+1 -1
View File
@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
mesh-catalog f3a57dcc56fb6c198095cafd11d0decff9f720c4 modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock
mesh-sdk a37181261c24a15b8092c06e11cce63e0e5f8ade conformance/events/module-event.json
mesh-sdk 2da0de75c3f0087cff9c2f5ef789ca6dc3c59045 conformance/events/module-event.json
To move them, from this repository's root, with the two repositories checked out beside it: