Compare commits

..
Author SHA1 Message Date
jschoubben 003eee8aa7 Issue every node's memberships when it is composed, so a module taken in with a new state is not refused it (issue 249 met again, issue 1945)
Release 6 removed the push (788fb681), and with it the only writer of the
memberships on ASSIGNMENTS. A module taken in since kept the membership the
old push last issued: mesh-delivery-gate, taken in with the state
"conditions" at 20:33 on 2026-10-25, was granted its bucket on the bus and
refused it by novox's tool runner every second, which keeps a module to its
membership (node-tools internal/bus/state.go issuedState).

A node's composition pass now issues the membership of each module assigned
to it, from the same records the grants are composed from, with the node's
private-network addresses and received contributions, before its
compositions are written. Only one that differs from the one last issued is
published, and only over that one (issues 930, 934). The memberships' stream,
every declared bucket and log are asserted first; the root-free judgement
runs only when a held seat promises a verified sender.

NATS-native piece moved (research 072, 03-domains, mesh-grants): the
membership is issued again by the take-in's pass, from the grants' records,
the step before mesh-grants takes both over.
2026-10-25 21:05:19 +01:00
125 changed files with 901 additions and 4304 deletions
+6 -42
View File
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
@@ -73,14 +72,6 @@ type actor struct {
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
renewEvery, poll time.Duration
// releasing is held for the whole of a release, so a second caller — the stop signal's and serve's own
// deferred one — waits until the lease is given back and its epoch recorded, never past it to the store's
// close (novox/hq issue 906).
releasing sync.Mutex
// keeper and inv are where a takeover is said (sayTakeover): set once the serving controller's condition
// store opens; nil before, and in a command.
keeper *conditions.Keeper
inv *inventory.Inventory
}
// theLease is this process's standing to the lease.
@@ -183,11 +174,8 @@ func (a *actor) forACommand(ctx context.Context) (uint64, error) {
return holder.Epoch, nil
}
// release gives back what this process holds, at its end — or at the stop signal, before the rest of the
// process is taken down (serveUnderTheLease). A second call waits for the first to finish.
// release gives back what this process holds, at its end.
func (a *actor) release() {
a.releasing.Lock()
defer a.releasing.Unlock()
a.mu.Lock()
closing := a.close
a.close = nil
@@ -278,40 +266,20 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
}
a.mu.Lock()
a.close = func() {
// Given back first, then recorded: an epoch is recorded released only when its key is gone, and
// whatever the gate says by then — a shutdown longer than half the key's age closed it, and the epoch
// was left open for the next holder to find "expired" (novox/hq issue 906, 2026-10-26).
if given := l.Release(context.Background()); given != 0 {
if held, err := l.Epoch(); err == nil {
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := inv.EndEpoch(ending, given, inventory.EpochReleased); err != nil {
say("how epoch %d ended could not be recorded: %v", given, err)
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
say("how epoch %d ended could not be recorded: %v", held, err)
}
cancel()
}
l.Release(context.Background())
js.Close()
}
a.mu.Unlock()
// **The stop signal gives the lease back first** (novox/hq issue 906): the moment ctx ends, before the rest
// of the process is taken down, the gate closes and the key goes, so the next controller takes the lease
// at once and the epoch is recorded released. Waiting for the process's own end left the key unrenewed for
// as long as the shutdown took — ten seconds on 2026-10-26, past the gate's half-age.
go func() {
<-ctx.Done()
if a.holds(l) {
say("stopping: the controller lease is given back first")
}
a.release()
}()
return gone, nil
}
// holds is whether this process holds l now.
func (a *actor) holds(l *lease.Lease) bool {
a.mu.Lock()
defer a.mu.Unlock()
return a.held == l
}
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
a.mu.Lock()
@@ -320,19 +288,15 @@ func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Invento
h := holderOf(instance)
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
Build: h.Build, Taken: time.Now()}, l.GivenBack())
Build: h.Build, Taken: time.Now()})
cancel()
if err != nil {
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
}
if given := l.GivenBack(); given != 0 {
fmt.Printf("the lease was given back by the controller of epoch %d: taken at once, at epoch %d\n", given, epoch)
}
for _, e := range expired {
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
}
a.sayTakeover(ctx, epoch)
go l.Keep(ctx)
go func() {
<-l.Lost()
@@ -78,8 +78,6 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
open := aMesh(t)
// Issue 1126: the placement export over this fresh mesh, before anything below changes it.
placementExportIsEachNodesPlacementFromTheStore(t, open)
ctx := t.Context()
raw, err := os.ReadFile("../../module.json")
if err != nil {
@@ -113,11 +111,6 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
t.Fatal(err)
}
// Seeded with the credential the controller already uses, so it is sealed: an account counts as issued only once
// sealed (ADR 0543 §2, PR #467), and genesis's seed is the credential in use.
if err := open.inventory.MarkBusAccountSealed(ctx, "anchor.mesh-controller"); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
t.Fatal(err)
}
+2 -8
View File
@@ -56,15 +56,9 @@ func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
}
}
// The real catalogue passes the command, the way it passes the test that used to be the only check: the catalogue this
// controller is released with, captured in testdata/beside at the commit CAPTURED names. Not the clone a merge check puts
// beside, which is the catalogue's trunk even in a delivery group holding a catalogue pull request (ADR 0484 clause 4):
// a controller whose check reads a word that group's catalogue brings would fail on the trunk's catalogue before the
// group merges both (issue 774: the release's sdk refuses an offer that states no credential form, and the catalogue's
// trunk states none until the release's catalogue merges). Moving CAPTURED to the catalogue a change is released with is
// part of that change.
// The real catalogue passes the command, the way it passes the test that used to be the only check.
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
root := filepath.Join(beside.CapturedDir(t, "mesh-catalog"), "modules")
root := beside.Catalogue(t)
paths, err := manifestsUnder(root)
if err != nil || len(paths) == 0 {
t.Fatalf("no manifests under %s: %v", root, err)
@@ -217,11 +217,6 @@ func TestTheAdaptersInputCarriesTheOwnSecretsTheBoxesTheLicencesAndTheBus(t *tes
Module: "objects"}, "a-password"); err != nil {
t.Fatal(err)
}
// Minted and sealed to its node: an account counts as issued, and is named among the minted, only once sealed
// (ADR 0543 §2, PR #467).
if err := inv.MarkBusAccountSealed(ctx, "anchor.objects"); err != nil {
t.Fatal(err)
}
if err := inv.PutBusMembership(ctx, "laptop", "sealed-membership"); err != nil {
t.Fatal(err)
}
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builderkeys"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
@@ -77,18 +76,13 @@ func aComposingMesh(t *testing.T) *composingMesh {
return m
}
// trunkKeys are a done key and the indexer's trunk value naming it. The indexer holds that value at the trunk's commit
// alone, as the real one does: answering it for any commit made a module registered at an older commit look already
// registered at the trunk's fingerprint, so the take-in only moved its commit and recorded no build (issue 1945).
// trunkKeys are a done key and the indexer's trunk value naming it.
type trunkKeys struct {
keyBuckets
trunk sdkdelivery.Module
}
func (k trunkKeys) ModuleAt(_ context.Context, _ string, commit string) (sdkdelivery.Module, error) {
if commit != "" && !sameCommitID(commit, k.trunk.Commit) {
return sdkdelivery.Module{}, builderkeys.ErrNoValue
}
func (k trunkKeys) ModuleAt(context.Context, string, string) (sdkdelivery.Module, error) {
return k.trunk, nil
}
+9
View File
@@ -88,6 +88,8 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma
}
return u.Policy(), u.From, u.Why
}
// The memberships of each node's modules are issued by its composition (memberships.go): read once for the pass.
issuing, issuingErr := membershipsFor(ctx, open, link.OverNATS{Conn: bus.Conn(), JS: bus.Context()})
sort.Strings(nodes)
out := map[string][]string{}
errsOf := map[string][]error{}
@@ -115,6 +117,13 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
return
}
// Its modules' memberships, before its compositions (issue 249): a failure is the node's, and its compositions
// are still written; the take-in that asked tries the node again.
if issuingErr != nil {
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, issuingErr))
} else if err := issuing.issue(ctx, node, declared); err != nil {
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
}
// The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror.
if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil {
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: its provisioner's plans: %w", node, err))
+2 -2
View File
@@ -23,10 +23,10 @@ func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
return p
}
objects := write("objects", `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"},"credential":{"form":"pair"}}],
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
resolver := write("resolver", `{"module":"resolver","version":"1",
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false,"credential":{"form":"none"}}]}`)
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
+2 -7
View File
@@ -182,9 +182,6 @@ func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssues
offer := catalogue.Offer{Name: "gitea-read", Scope: catalogue.ScopeMesh}
if issued {
offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}
} else {
// A login of each consumer's own, said: an offer naming no form has no credential (mesh-sdk 6b8c694).
offer.Credential = &catalogue.OfferCredential{Form: catalogue.CredentialFormPair}
}
return catalogue.Manifest{Module: "gitea", Version: "1", Moved: moved, Provides: []catalogue.Offer{offer}}
}
@@ -219,10 +216,8 @@ func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssues
// marked moved; another module is not.
func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) {
vault := catalogue.Manifest{Module: "mesh-vault", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-vault", Scope: catalogue.ScopeMesh}},
Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh,
Credential: &catalogue.OfferCredential{Form: catalogue.CredentialFormPair}}}}
other := catalogue.Manifest{Module: "mesh-vault", Version: "1", Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh,
Credential: &catalogue.OfferCredential{Form: catalogue.CredentialFormPair}}}}
Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh}}}
other := catalogue.Manifest{Module: "mesh-vault", Version: "1", Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh}}}
if got := needOf(t, map[string]catalogue.Manifest{"mesh-vault": vault}, "mesh-vault", "gitea", "secret"); !got.IssuedToHolder || !got.ProviderMoved {
t.Errorf("the vault's consumer is not answered by a moved provider: %+v", got)
}
+2 -15
View File
@@ -89,28 +89,15 @@ func TestTwoControllersOneActs(t *testing.T) {
t.Fatalf("B, waiting, may act: %v", err)
}
// A is stopped, as its service manager stops a controller being replaced (novox/hq issue 906): the stop
// alone gives the lease back, before anything else of A is taken down, and B takes it within a second, at
// a higher epoch — no wait for the key's age, and no condition said.
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
stopA()
stopped := time.Now()
a.release()
var lostB <-chan struct{}
select {
case lostB = <-tookB:
case <-time.After(10 * time.Second):
t.Fatal("B did not take the lease A gave back")
}
if waited := time.Since(stopped); waited > time.Second {
t.Fatalf("B took the lease %s after A was stopped: the stop did not give it back first", waited)
}
a.release() // what serve's own end does after: waits for the release under way, gives nothing twice
conditionsOfB := conditions.NewInMemory()
keeperOfB := conditions.NewKeeper(ctx, conditions.Options{Store: conditionsOfB, History: conditionsOfB})
b.watchTakeovers(ctx, inv, keeperOfB)
keeperOfB.Close(ctx)
if said, err := conditionsOfB.Since(ctx, stopped.Add(-time.Minute)); err != nil || len(said) != 0 {
t.Fatalf("a lease given back is said as a condition: %+v, %v", said, err)
}
select {
case <-lostA:
default:
+186
View File
@@ -0,0 +1,186 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// **A node's composition issues its modules' memberships** (novox/hq ADR 0160, ADR 0201; issue 249 met again under
// issue 1945).
//
// A membership is what an assignment serves and reaches: its subjects, its seats, and the state, logs, streams and waits
// its manifest declares, published last-per-subject on the AssignmentsStream, where the tool runner and every bus process
// read and follow their own. A push issued them before it sent a declaration; release 6 removed the push and nothing
// issued them after, so a module taken in with a new state, read or stream was granted it on the bus and refused it by its
// runtime, which keeps a module to the membership last issued (2026-10-25: mesh-delivery-gate's `conditions`, on novox
// from 20:33). Composing a node is now where its memberships are issued: from the same records the bus's grants are
// composed from (so what a runtime serves and what its account may are one composition), with the private network's
// addresses and what each module receives from the node's composition, before its compositions are written (the code
// arriving first was refused its own state, issue 249).
//
// Only a membership that differs from the one last issued is published, and only over that one, so a pass that changes
// nothing re-serves nothing and a newer one issued meanwhile is not overwritten (issues 930, 934).
// membershipPass is what one composer pass issues every node's memberships from, read once for the pass.
type membershipPass struct {
records broker.Records
where broker.Placements
bus link.OverNATS
}
// membershipsFor reads the records, asserts the memberships' stream and every declared bucket and log (a membership
// names them, and a module whose state does not exist fails its first read), and judges which machines are root-free
// (ADR 0259 §8: a channel's verified sender is composed only from one) when a seat promises one, as a push did.
func membershipsFor(ctx context.Context, open *stores, bus link.OverNATS) (*membershipPass, error) {
records, err := open.inventory.BusRecords(ctx)
if err != nil {
return nil, fmt.Errorf("the memberships are not issued: %w", err)
}
raiser := broker.OnConn(bus.Conn)
for _, s := range broker.MeshStreams() {
if s.Name != broker.AssignmentsStream {
continue
}
if err := raiser.EnsureStream(s); err != nil {
return nil, fmt.Errorf("the memberships are not issued: their stream: %w", err)
}
}
buckets, err := open.inventory.DeclaredBuckets(ctx)
if err != nil {
return nil, fmt.Errorf("the memberships are not issued: the modules' state: %w", err)
}
if _, err := broker.RaiseBuckets(raiser, buckets); err != nil {
return nil, fmt.Errorf("the memberships are not issued: the modules' state could not be asserted: %w", err)
}
logs, err := open.inventory.DeclaredLogs(ctx)
if err != nil {
return nil, fmt.Errorf("the memberships are not issued: the modules' logs: %w", err)
}
if _, err := broker.RaiseLogs(raiser, logs); err != nil {
return nil, fmt.Errorf("the memberships are not issued: the modules' logs could not be asserted: %w", err)
}
var whys map[string]string
if promisesVerifiedSender(records) {
records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn),
records.Nodes, time.Now()))
}
where := broker.PlacementsOf(records, records.Interchangeable)
for _, line := range droppedVerifiedSenders(records, where, whys) {
fmt.Println(line)
}
return &membershipPass{records: records, where: where, bus: bus}, nil
}
// promisesVerifiedSender says a seat some module holds promises a verified sender: only then is the root-free judgement,
// a discovery on the bus, worth its seconds. Pure.
func promisesVerifiedSender(records broker.Records) bool {
for _, declared := range records.Assigned {
for _, d := range declared {
for _, s := range d.Holds {
if slices.Contains(s.Capabilities, "verified-sender") {
return true
}
}
}
}
return false
}
// issue issues node's memberships on the bus, from its composition's addresses and received contributions.
func (p *membershipPass) issue(ctx context.Context, node string, declared sendable) error {
issued, err := issueComposedMemberships(node, p.records, p.where, declared.Mesh, declared.Received,
func(node, module string) ([]byte, uint64, error) {
m, err := p.bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module), nats.Context(ctx))
if err != nil {
return nil, 0, err
}
return m.Data, m.Sequence, nil
},
func(node, module string, body []byte, seq uint64) error {
return p.bus.PublishMembershipAfter(ctx, node, module, body, seq)
})
if len(issued) > 0 {
fmt.Printf("memberships: %s: issued %s\n", node, strings.Join(issued, ", "))
}
return err
}
// issueComposedMemberships composes the membership of every module assigned to node and publishes each one that differs
// from the one last issued, over that one (seq 0 when none was: only if none was issued meanwhile). It answers the
// modules it issued; every failure is tried past and joined in the error.
func issueComposedMemberships(node string, records broker.Records, where broker.Placements, mesh []string,
received map[string]map[string][]catalogue.Contribution,
last func(node, module string) ([]byte, uint64, error),
publish func(node, module string, body []byte, seq uint64) error) ([]string, error) {
var issued []string
var errs []error
for _, d := range records.Assigned[node] {
body, err := composedMembership(node, d, where, mesh, received[d.Module])
if err != nil {
errs = append(errs, fmt.Errorf("%s on %s: its membership: %w", d.Module, node, err))
continue
}
was, seq, err := last(node, d.Module)
switch {
case errors.Is(err, nats.ErrMsgNotFound):
was, seq = nil, 0
case err != nil:
errs = append(errs, fmt.Errorf("%s on %s: its membership as last issued could not be read: %w", d.Module, node, err))
continue
}
if sameMembership(was, body) {
continue
}
if err := publish(node, d.Module, body, seq); err != nil {
errs = append(errs, fmt.Errorf("%s on %s: its membership could not be issued: %w", d.Module, node, err))
continue
}
issued = append(issued, d.Module)
}
sort.Strings(issued)
return issued, errors.Join(errs...)
}
// composedMembership is one assignment's membership as a node's composition issues it: MembershipFor, with the private
// network's addresses and what the module receives, by requirement (ADR 0167).
func composedMembership(node string, d broker.Declared, where broker.Placements, mesh []string,
received map[string][]catalogue.Contribution) ([]byte, error) {
m := broker.MembershipFor(node, d, where)
m.Mesh = mesh
for requirement, given := range received {
raw, err := json.Marshal(given)
if err != nil {
return nil, err
}
if m.Receives == nil {
m.Receives = map[string]json.RawMessage{}
}
m.Receives[requirement] = raw
}
return json.Marshal(m)
}
// sameMembership says two memberships say the same, whatever the order of their fields. Pure.
func sameMembership(was, now []byte) bool {
if was == nil {
return false
}
var a, b any
if json.Unmarshal(was, &a) != nil || json.Unmarshal(now, &b) != nil {
return false
}
return reflect.DeepEqual(a, b)
}
+1 -6
View File
@@ -133,14 +133,9 @@ func nodeCommand(ctx context.Context, args []string) error {
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
return nodeJudgeDirectories(ctx, open, args[1:])
case "placement-export":
// Each node's placement as the catalogue's placement file says it (novox/hq ADR 0555 §6): read-only, so a verb
// may run it.
return placementExport(ctx, open, args[1:])
default:
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
"setuid-search, judge-directories and placement-export", args[0])
"setuid-search and judge-directories", args[0])
}
}
-50
View File
@@ -1,50 +0,0 @@
package main
import (
"context"
"errors"
"sort"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// placementExport prints each node's placement as the catalogue's placement file says it (novox/hq ADR 0555 §6, issue
// 1126): what a person decided of the node and where its modules run, read by this controller's own code from its own
// store, as JSON keyed by node. The one verb the controller gains for its exit (the operator, 2026-10-26 19:06:40, ADR
// 0547 §1's one exception besides §5's grant): read once by the seed of the placement files, whose pull request merges
// only when its composition check composes every node equal to what it runs. It reads the same Input every composition
// of this controller reads (readMeshAndNodes, readSettings) and writes nothing.
//
// It moves a piece of the research "The controller, NATS-native": the controller's last node data (node rows, placement,
// ports, pins, bindings, config placement) leaves it for the catalogue, before its exit at ADR 0547's third milestone.
func placementExport(ctx context.Context, open *stores, args []string) error {
if len(args) > 1 {
return errors.New("node placement-export [<name>]")
}
in, _, err := readMeshAndNodes(ctx, open)
if err != nil {
return err
}
if _, err := readSettings(ctx, open, &in); err != nil {
return err
}
var names []string
if len(args) == 1 {
names = []string{args[0]}
} else {
for _, n := range in.Nodes {
names = append(names, n.Name)
}
sort.Strings(names)
}
out := make(map[string]delivery.Placement, len(names))
for _, name := range names {
p, err := frombus.Export(in, name)
if err != nil {
return err
}
out[name] = p
}
return printJSON(out)
}
@@ -1,71 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
)
// Issue 1126 (novox/hq ADR 0555 §6, the operator's one export verb of 2026-10-26 19:06:40): `node placement-export`
// answers, from the real store, each node's placement as the catalogue's placement file says it: what a person decided
// of the node (its endpoint, site, hub and address on the private network) and its modules in the order the composition
// reads them, exactly frombus.Export of the input this controller composes from; one node when named; a node the store
// lacks is refused. Checked within TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker, over the same fresh
// mesh, so the export adds no integration test of its own (the test budget).
func placementExportIsEachNodesPlacementFromTheStore(t *testing.T, open *stores) {
t.Helper()
ctx := t.Context()
in, _, err := readMeshAndNodes(ctx, open)
if err != nil {
t.Fatal(err)
}
if _, err := readSettings(ctx, open, &in); err != nil {
t.Fatal(err)
}
var all map[string]sdkdelivery.Placement
if err := json.Unmarshal([]byte(stdoutOf(t, func() error { return placementExport(ctx, open, nil) })), &all); err != nil {
t.Fatal(err)
}
if len(all) != 2 {
t.Fatalf("exported %d nodes, want anchor and laptop", len(all))
}
for i, name := range []string{"anchor", "laptop"} {
got := all[name]
d := got.Decided
if d.Endpoint != name+".example:51820" || d.Site != "here" || d.Hub != (name == "anchor") || d.Address != []string{"10.77.0.1", "10.77.0.2"}[i] {
t.Errorf("%s's decided facts are %+v", name, d)
}
if a := got.Assigned(); len(a) == 0 || !strings.Contains(strings.Join(a, ","), overlay.Name) {
t.Errorf("%s's modules are %v, want the private network among them", name, a)
}
// What it prints is the file the seed writes: at the placement file's version, read back by its reader.
if raw, _ := json.Marshal(got); got.V != sdkdelivery.Version {
t.Errorf("%s is exported at version %d: %s", name, got.V, raw)
} else if _, err := sdkdelivery.DecodePlacement(raw); err != nil {
t.Errorf("%s's export does not read as a placement file: %v", name, err)
}
want, err := frombus.Export(in, name)
if err != nil {
t.Fatal(err)
}
if !sdkdelivery.SamePlacement(got, want) {
t.Errorf("%s's export differs from frombus.Export of the controller's own input:\n%+v\n%+v", name, got, want)
}
}
var one map[string]sdkdelivery.Placement
if err := json.Unmarshal([]byte(stdoutOf(t, func() error { return placementExport(ctx, open, []string{"laptop"}) })), &one); err != nil {
t.Fatal(err)
}
if _, ok := one["laptop"]; !ok || len(one) != 1 {
t.Errorf("one node named exported %v", one)
}
if err := placementExport(ctx, open, []string{"stranger"}); err == nil {
t.Error("a node the store lacks was exported")
}
}
-3
View File
@@ -180,9 +180,6 @@ func serve(ctx context.Context) (err error) {
if keeper != nil {
defer closeConditions(keeper)
defer stopRunning() // on an error return, the goroutines end before the keeper closes
// The takeover of the lease said as it happened, raised and cleared (novox/hq issue 906).
theLease.watchTakeovers(ctx, inv, keeper)
defer theLease.watchTakeovers(ctx, nil, nil) // the keeper closes after this: nothing said into it then
}
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
@@ -108,14 +108,83 @@ func TestADroppedVerifiedSenderIsSaidWithItsReason(t *testing.T) {
}
}
func TestAMembershipNeverIssuedIsLeftToItsOwnSend(t *testing.T) {
records, _ := verifiedSenderRecords(map[string]bool{"novox": true})
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
var said []string
n := reissueVerifiedSenders(records, broker.PlacementsOf(records, nil), bus.get, bus.put,
func(f string, a ...any) { said = append(said, f) })
if n != 0 || len(said) != 0 {
t.Fatalf("issued %d, said %v", n, said)
// **A node's composition issues every membership that changed** (issue 249 met again under issue 1945): release 6 removed
// the push that issued memberships, and mesh-delivery-gate taken in with a new state `conditions` was granted its bucket and
// refused it by the tool runner, which reads the membership last issued (2026-10-25, novox). The self-check's reissue
// leaves one never issued alone: the composition issues it.
func TestAComposedNodeIssuesEveryMembershipThatChanged(t *testing.T) {
gate := func(states ...string) broker.Declared {
d := broker.Declared{Module: "mesh-delivery-gate"}
for _, s := range states {
d.State = append(d.State, broker.Bucket{Module: "mesh-delivery-gate", Name: s})
}
return d
}
records := func(d broker.Declared) broker.Records {
return broker.Records{Nodes: []string{"novox"}, Assigned: map[string][]broker.Declared{"novox": {d}}}
}
issuedAs := func(d broker.Declared, mesh []string) []byte {
body, err := composedMembership("novox", d, broker.PlacementsOf(records(d), nil), mesh, nil)
if err != nil {
t.Fatal(err)
}
return body
}
newer := []byte(`{"node":"novox","module":"mesh-delivery-gate","newer":true}`)
for _, c := range []struct {
name string
before []byte // the membership last issued, nil for none
between bool // another issued meanwhile, after the read
issued []string
states []string // what the bus holds after, by state name; nil when it must hold newer
err string
}{
{name: "never issued is issued", issued: []string{"mesh-delivery-gate"}, states: []string{"asks", "conditions"}},
{name: "a new state is issued", before: issuedAs(gate("asks"), []string{"10.0.0.1"}),
issued: []string{"mesh-delivery-gate"}, states: []string{"asks", "conditions"}},
{name: "unchanged is not issued again", before: issuedAs(gate("asks", "conditions"), []string{"10.0.0.1"}),
states: []string{"asks", "conditions"}},
{name: "a newer one meanwhile is not overwritten", before: issuedAs(gate("asks"), []string{"10.0.0.1"}),
between: true, err: "wrong last sequence"},
} {
t.Run(c.name, func(t *testing.T) {
r := records(gate("asks", "conditions"))
where := broker.PlacementsOf(r, nil)
bus := &fakeBus{last: map[string][]byte{}, published: map[string][]byte{}, seq: map[string]uint64{}}
if c.before != nil {
bus.last["novox/mesh-delivery-gate"], bus.seq["novox/mesh-delivery-gate"] = c.before, 1
}
if c.between {
bus.between = func() { bus.last["novox/mesh-delivery-gate"], bus.seq["novox/mesh-delivery-gate"] = newer, 2 }
}
if n := reissueVerifiedSenders(r, where, bus.get, bus.put, func(string, ...any) {}); n != 0 {
t.Fatalf("the self-check's reissue issued %d", n)
}
issued, err := issueComposedMemberships("novox", r, where, []string{"10.0.0.1"}, nil, bus.get, bus.put)
if c.err == "" && err != nil || c.err != "" && (err == nil || !strings.Contains(err.Error(), c.err)) {
t.Fatalf("error %v, want %q", err, c.err)
}
if strings.Join(issued, ",") != strings.Join(c.issued, ",") {
t.Fatalf("issued %v, want %v", issued, c.issued)
}
if c.states == nil {
if string(bus.last["novox/mesh-delivery-gate"]) != string(newer) {
t.Fatalf("the newer membership was overwritten: %s", bus.last["novox/mesh-delivery-gate"])
}
return
}
var m broker.Membership
if err := json.Unmarshal(bus.last["novox/mesh-delivery-gate"], &m); err != nil {
t.Fatal(err)
}
var names []string
for _, s := range m.State {
names = append(names, s.Name)
}
if strings.Join(names, ",") != strings.Join(c.states, ",") || strings.Join(m.Mesh, ",") != "10.0.0.1" {
t.Fatalf("the bus holds states %v and mesh %v, want %v", names, m.Mesh, c.states)
}
})
}
}
+1 -8
View File
@@ -349,11 +349,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err
}
return []string{"node", "judge-directories", str("node")}, nil
case "placement-export":
if node := str("node"); node != "" {
return []string{"node", "placement-export", node}, nil
}
return []string{"node", "placement-export"}, nil
case "modules":
return []string{"module", "list", "--json"}, nil
case "seats":
@@ -1405,9 +1400,7 @@ func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
var nodeReads = map[string]bool{"list": true, "show": true,
// Read-only on the node too: the engine judges its directories and changes nothing (novox/hq issue 358).
"judge-directories": true,
// Each node's placement as the catalogue's placement file says it, read from the store (novox/hq ADR 0555 §6).
"placement-export": true}
"judge-directories": true}
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
// with no subcommands every word is a flag, its value or a name it reads.
+15 -113
View File
@@ -1,7 +1,6 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
@@ -148,10 +147,9 @@ var signalsTable = []signalRow{
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
}},
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
Bound: "15 s (the key's age); a holder that lost the lease or stopped renewing it is said once by the " +
"controller that took it over, raised and cleared as it takes it, and for an hour after while none " +
"holds it; a lease given back is never said; a lease bucket found raised again from nothing is said " +
"for an hour after; a controller serving without the lease, for as long as it does",
Bound: "15 s (the key's age); a holder that lost the lease, or stopped renewing and was taken over, and a " +
"lease bucket found raised again from nothing, are said for an hour after; a controller serving without " +
"the lease, for as long as it does",
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
@@ -646,29 +644,10 @@ func watchLease(f *signalFacts) []conditions.Observation {
"does not know it lost it",
Said: l.resetSaid})
}
// An epoch below the one this controller holds was taken over by it: said once by the takeover itself,
// raised and cleared (sayTakeover), not observed for an hour after (novox/hq issue 906). One that ended
// while this controller does not hold the lease — serving without it — is still said for the hour.
var ended []inventory.Epoch
for _, e := range l.ended {
if l.held && e.Epoch < l.epoch {
continue
}
ended = append(ended, e)
}
if o, ok := lostObservation(ended, l.epoch, f.now); ok {
out = append(out, o)
}
return out
}
// lostObservation is the lease-lost condition of the epochs that lost the lease or let it expire within the
// hour, named by the newest; false when none did. An epoch given back is no loss.
func lostObservation(ended []inventory.Epoch, acting uint64, now time.Time) (conditions.Observation, bool) {
var lost []string
newest := inventory.Epoch{}
for _, e := range ended {
if e.Ended == nil || e.How == inventory.EpochReleased || now.Sub(*e.Ended) > advisoryQuiet {
for _, e := range l.ended {
if e.Ended == nil || e.How == inventory.EpochReleased || f.now.Sub(*e.Ended) > advisoryQuiet {
continue
}
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
@@ -677,95 +656,18 @@ func lostObservation(ended []inventory.Epoch, acting uint64, now time.Time) (con
newest = e
}
}
if len(lost) == 0 {
return conditions.Observation{}, false
}
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
if newest.How == inventory.EpochExpired {
how = "stopped renewing it without giving it back, and was taken over"
}
return conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent, Source: "S12",
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
newest.Instance, how, acting),
Said: strings.Join(lost, "; ")}, true
}
// takeover is the condition a takeover says: the epoch just before the one taken, when it lost the lease or
// let it expire; false when it gave it back, or none is recorded.
func takeover(ended []inventory.Epoch, taken uint64, now time.Time) (conditions.Observation, bool) {
var before *inventory.Epoch
for i, e := range ended {
if e.Epoch < taken && (before == nil || e.Epoch > before.Epoch) {
before = &ended[i]
if len(lost) > 0 {
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
if newest.How == inventory.EpochExpired {
how = "stopped renewing it without giving it back, and was taken over"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
newest.Instance, how, l.epoch),
Said: strings.Join(lost, "; ")})
}
if before == nil {
return conditions.Observation{}, false
}
return lostObservation([]inventory.Epoch{*before}, taken, now)
}
// sayTakeover is the takeover said as an event (novox/hq issue 906): when the controller before this one lost
// the lease or let it expire, the lease-lost condition is raised and cleared at once by this controller, which
// holds the lease — the operator reads that it happened and that it is over, not an urgent alarm for an hour.
// A lease given back says nothing. Said once the condition store is open; a take before that is said then.
func (a *actor) sayTakeover(ctx context.Context, epoch uint64) {
a.mu.Lock()
keeper, inv := a.keeper, a.inv
a.mu.Unlock()
if keeper == nil || inv == nil || epoch == 0 {
return
}
now := time.Now()
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
ended, err := inv.EpochsSince(reading, now.Add(-advisoryQuiet))
if err != nil {
// Failure is loud: S12 no longer observes an epoch below the one held, so an unread takeover is said as
// one that was not a handover — raised, and left for S12 to clear (ADR 0558 §3).
fmt.Printf("how the epoch before %d ended cannot be read, so its takeover is said as an expiry: %v\n", epoch, err)
o := conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
Kind: "lease-lost", Severity: conditions.Urgent, Source: "S12",
Summary: fmt.Sprintf("the controller of epoch %d took the lease, and how the controller before it ended "+
"cannot be read: it is said as one that stopped renewing without giving it back", epoch),
Said: err.Error()}
if _, err := keeper.Observe(reading, o); err != nil {
fmt.Printf("the takeover at epoch %d could not be said: %v\n", epoch, err)
}
return
}
sayTakeoverOf(reading, keeper, ended, epoch, now)
}
// sayTakeoverOf raises and clears the takeover's condition over the epochs read.
func sayTakeoverOf(ctx context.Context, keeper *conditions.Keeper, ended []inventory.Epoch, epoch uint64, now time.Time) {
o, ok := takeover(ended, epoch, now)
if !ok {
return
}
if _, err := keeper.Observe(ctx, o); err != nil {
fmt.Printf("the takeover at epoch %d could not be said: %v\n", epoch, err)
return
}
if _, err := keeper.Clear(ctx, o.Key(), fmt.Sprintf("taken over: the controller of epoch %d holds the lease", epoch)); err != nil {
fmt.Printf("the takeover at epoch %d was said and could not be cleared: %v\n", epoch, err)
}
}
// watchTakeovers gives the serving controller's condition store to the lease, and says the takeover of the
// epoch it holds, if it took one over before the store opened.
func (a *actor) watchTakeovers(ctx context.Context, inv *inventory.Inventory, keeper *conditions.Keeper) {
a.mu.Lock()
a.keeper, a.inv = keeper, inv
held := a.held
a.mu.Unlock()
if held == nil {
return
}
if epoch, err := held.Epoch(); err == nil {
a.sayTakeover(ctx, epoch)
}
return out
}
// handActsWithin is how far back a repeated cause counts (S15).
+1 -32
View File
@@ -491,41 +491,13 @@ func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
}
}
// **The lease lost is said for an hour while nobody took it over, a takeover is said once as it happens —
// raised and cleared by the controller that took it — a lease given back is never said, and serving without
// it for as long as it lasts** (S12, novox/hq issue 906).
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
now := time.Now()
f := calm(now)
expired := now.Add(-59 * time.Minute)
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
Ended: &expired, How: inventory.EpochExpired}}
// Held at epoch 57, above it: the takeover said it, and no hour of observation follows.
if got := watchLease(f); len(got) != 0 {
t.Fatalf("an epoch this controller took over is observed after the takeover: %+v", got)
}
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Now: func() time.Time { return now }})
sayTakeoverOf(t.Context(), k, f.lease.ended, 57, now)
if open, err := k.Open(t.Context()); err != nil || len(open) != 0 {
t.Fatalf("the takeover is left open: %+v, %v", open, err)
}
k.Close(t.Context()) // its transitions written to the history
history, err := store.Since(t.Context(), now.Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
var changes []string
for _, e := range history {
if e.Key == "core.controller.lease.lost" {
changes = append(changes, e.Change)
}
}
if !slices.Equal(changes, []string{"raised", "cleared"}) {
t.Fatalf("the takeover of an expired lease is not raised and cleared: %v", changes)
}
// Not holding the lease, the loss is said for the hour.
f.lease.held = false
got := watchLease(f)
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
got[0].Severity != conditions.Urgent {
@@ -540,9 +512,6 @@ func TestALeaseLostOrMissingIsSaid(t *testing.T) {
if got := watchLease(f); len(got) != 0 {
t.Fatalf("a lease given back is said as lost: %+v", got)
}
if _, said := takeover(f.lease.ended, 57, now); said {
t.Fatal("the takeover of a lease given back is said")
}
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+2 -11
View File
@@ -53,16 +53,6 @@ func Dir(t testing.TB, repository string) string {
t.Logf("judged against %s as cloned beside this check", dir)
return dir
}
t.Logf("no %s beside this check: a merge check judges %s against its own clone", Env, repository)
return CapturedDir(t, repository)
}
// CapturedDir is the copy of the named repository captured in testdata/beside, laid out as Dir lays it out, whatever
// MESH_CHECK_BESIDE says: for a test that judges this repository against the commit it was released with, which a
// merge check does not clone (it clones the catalogue at its trunk even in a delivery group holding a catalogue pull
// request, ADR 0484 clause 4).
func CapturedDir(t testing.TB, repository string) string {
t.Helper()
from := filepath.Join(Captured(), repository)
if _, err := os.Stat(from); err != nil {
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
@@ -91,7 +81,8 @@ func CapturedDir(t testing.TB, repository string) string {
if err != nil {
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
}
t.Logf("judged against the copy of %s captured in testdata/beside (see CAPTURED)", repository)
t.Logf("judged against the copy of %s captured in testdata/beside (see CAPTURED); a merge check "+
"judges it against the repository's own clone", repository)
return dir
}
+7 -9
View File
@@ -46,8 +46,9 @@ func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
t.Fatalf("a consumer made from now holds %d of the stream's past (policy %v)", info.NumPending, info.Config.DeliverPolicy)
}
_, _ = js.js.Publish("history.merged", []byte("new"))
if got := pendingOf(js, stream, "fresh", 1); got != 1 {
t.Fatalf("a new announcement is not pending: %d", got)
time.Sleep(100 * time.Millisecond)
if info, _ = js.js.ConsumerInfo(stream, "fresh"); info.NumPending != 1 {
t.Fatalf("a new announcement is not pending: %d", info.NumPending)
}
// Asserted again, it keeps where it is.
if err := js.EnsureConsumer(fresh); err != nil {
@@ -60,8 +61,8 @@ func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
if err := js.EnsureConsumer(old); err != nil {
t.Fatal(err)
}
if got := pendingOf(js, stream, "old", 6); got != 6 {
t.Fatalf("the default should replay all six: %d", got)
if info, _ = js.js.ConsumerInfo(stream, "old"); info.NumPending != 6 {
t.Fatalf("the default should replay all six: %d", info.NumPending)
}
before, after, err := js.ResetConsumer(stream, "old")
if err != nil {
@@ -88,13 +89,10 @@ func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
}
}
// pendingOf polls a consumer's pending count until it is want or ten seconds have passed, and returns what it saw
// last. The server counts a consumer's pending after it acknowledges the publish, not before, so a test that reads
// the count at once fails on a loaded build node (a try of main on ace, 2026-10-26): every test that expects a
// count after a publish reads it through here.
// pendingOf polls a consumer's pending count until it is want or a second has passed, and returns what it saw last.
func pendingOf(js *JetStream, stream, name string, want uint64) uint64 {
var got uint64
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(20 * time.Millisecond) {
for deadline := time.Now().Add(time.Second); ; time.Sleep(20 * time.Millisecond) {
if info, err := js.js.ConsumerInfo(stream, name); err == nil {
got = info.NumPending
}
@@ -73,15 +73,15 @@ func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testin
if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil {
t.Fatal(err)
}
if got := pendingOf(js, EventsStream, MergesConsumerName, 1); got != 1 {
t.Fatalf("the merges consumer holds %d pending; want the merge", got)
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
if err != nil || merges.NumPending != 1 {
t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err)
}
if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil {
t.Fatal(err)
}
pending := pendingOf(js, EventsStream, MergesConsumerName, 1)
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
if err != nil || pending != 1 || merges.Delivered.Stream != 0 {
merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 {
t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err)
}
}
@@ -107,8 +107,9 @@ func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T
if err := AssertMeshConsumers(js); err != nil {
t.Fatal(err)
}
if got := pendingOf(js, EventsStream, MergesConsumerName, 1); got != 1 {
t.Fatalf("the merges consumer holds %d pending; want the merge the old consumer never acknowledged", got)
merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
if err != nil || merges.NumPending != 1 {
t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err)
}
}
-106
View File
@@ -1,106 +0,0 @@
package broker
import "testing"
// Defends issue 1790 and novox/hq ADR 0547 §5 (the operator, 2026-10-26 08:54): the grants part may create the stream
// of each bucket, log and stream a placed module's manifest declares, by its own name, and read it back; nothing
// wider, nothing updated or deleted, and no other principal so. Held byte-equal in the controller's copy. And ADR 0547
// clause 7 (issue 1945, the operator's second exception): the grants part writes its own object bucket of sealed user
// lists, and the node-engine of the node holding mesh-broker alone reads its one key and an object of it by name.
func TestTheGrantsPartCreatesOnlyDeclaredStreams(t *testing.T) {
records := Records{
Nodes: []string{"novox", "ace"},
Assigned: map[string][]Declared{"ace": {{Module: "plex"}}, "novox": {
{Module: "nats", SnapshotsTheBus: true},
{Module: "mesh-delivery-builder",
State: []Bucket{{Module: "mesh-delivery-builder", Name: "steps", History: 5}},
Logs: []Log{{Module: "mesh-delivery-builder", Name: "builds"}},
Streams: []ModuleStream{{Module: "mesh-delivery-builder", Name: "BUILDASKS", Events: []string{"ask"}}}},
{Module: GrantsModule, State: []Bucket{{Module: GrantsModule, Name: "assertions"}}},
}},
}
users, err := Users(records)
if err != nil {
t.Fatal(err)
}
perms := map[string]Permissions{}
for _, u := range users {
if u.Kind == KindNode {
p, err := PermissionsFor(u)
if err != nil {
t.Fatalf("%s: %v", u.Username(), err)
}
perms[u.Username()] = p
continue
}
if u.Kind != KindModule {
continue
}
p, err := PermissionsFor(u)
if err != nil {
t.Fatalf("%s: %v", u.Username(), err)
}
perms[u.Module] = p
}
for _, row := range []struct {
who, subject string
may bool
}{
{GrantsModule, "$JS.API.STREAM.CREATE.KV_mesh-delivery-builder_steps", true},
{GrantsModule, "$JS.API.STREAM.INFO.KV_mesh-delivery-builder_steps", true},
{GrantsModule, "$JS.API.STREAM.CREATE.LOG_mesh-delivery-builder_builds", true},
{GrantsModule, "$JS.API.STREAM.CREATE.BUILDASKS", true},
{GrantsModule, "$JS.API.STREAM.CREATE.KV_mesh-grants_assertions", true},
{GrantsModule, "$JS.API.STREAM.UPDATE.KV_mesh-delivery-builder_steps", false},
{GrantsModule, "$JS.API.STREAM.DELETE.KV_mesh-delivery-builder_steps", false},
{GrantsModule, "$JS.API.STREAM.PURGE.KV_mesh-delivery-builder_steps", false},
{GrantsModule, "$JS.API.STREAM.CREATE.KV_mesh-delivery-builder_undeclared", false},
{GrantsModule, "$JS.API.STREAM.CREATE.EVENTS", false},
{GrantsModule, "$JS.API.STREAM.CREATE.ASSIGNMENTS", false},
{GrantsModule, "$JS.API.STREAM.INFO.EVENTS", false},
{"mesh-delivery-builder", "$JS.API.STREAM.CREATE.KV_mesh-delivery-builder_steps", false},
// The sealed user list (ADR 0547 clause 7): the grants part writes its own bucket and no other.
{GrantsModule, "$JS.API.STREAM.CREATE.OBJ_mesh-grants-user-list", true},
{GrantsModule, "$O.mesh-grants-user-list.C.abc", true},
{GrantsModule, "$JS.API.STREAM.PURGE.OBJ_mesh-grants-user-list", true},
{GrantsModule, "$O.declaration-files-novox.C.abc", false},
{GrantsModule, "$JS.API.STREAM.DELETE.OBJ_mesh-grants-user-list", false},
{"mesh-delivery-builder", "$O.mesh-grants-user-list.C.abc", false},
// The bus node's engine reads the one key and an object by name; nothing of the bucket's list, no write.
{"node.novox", "$JS.API.DIRECT.GET.KV_mesh-grants_user-list.$KV.mesh-grants_user-list.sealed", true},
{"node.novox", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_user-list.abc.$KV.mesh-grants_user-list.sealed", true},
{"node.novox", "$JS.API.DIRECT.GET.OBJ_mesh-grants-user-list.$O.mesh-grants-user-list.M.abc", true},
{"node.novox", "$JS.API.CONSUMER.CREATE.OBJ_mesh-grants-user-list.abc.$O.mesh-grants-user-list.C.abc", true},
{"node.novox", "$JS.API.DIRECT.GET.KV_mesh-grants_user-list.$KV.mesh-grants_user-list.list", false},
{"node.novox", "$JS.API.STREAM.INFO.OBJ_mesh-grants-user-list", false},
{"node.novox", "$JS.API.STREAM.MSG.GET.OBJ_mesh-grants-user-list", false},
{"node.novox", "$O.mesh-grants-user-list.C.abc", false},
{"node.ace", "$JS.API.DIRECT.GET.KV_mesh-grants_user-list.$KV.mesh-grants_user-list.sealed", false},
{"node.ace", "$JS.API.DIRECT.GET.OBJ_mesh-grants-user-list.$O.mesh-grants-user-list.M.abc", false},
} {
if got := MayPublish(perms[row.who], row.subject); got != row.may {
t.Errorf("%s may publish %s: %v, want %v", row.who, row.subject, got, row.may)
}
}
// A name that is no plain token is never granted, and a module other than the grants part is never granted one.
if got := CreateStreamGrants([]string{"*", ">", "KV_a.b", "KV_a_b"}); len(got) != 2 {
t.Errorf("grants for wildcard names: %v", got)
}
other, err := PermissionsFor(Principal{Kind: KindModule, Node: "novox", Module: "shop", CreatesStreams: []string{"KV_shop_x"}})
if err != nil {
t.Fatal(err)
}
if MayPublish(other, "$JS.API.STREAM.CREATE.KV_shop_x") {
t.Error("a module other than the grants part was granted a create")
}
// The writers table refuses the grants part an update, and a create of a stream it was not given.
g := Principal{Kind: KindModule, Node: "novox", Module: GrantsModule, CreatesStreams: []string{"KV_a_b"}}
if err := CheckWriters(g, []string{"$JS.API.STREAM.CREATE.KV_a_b"}); err != nil {
t.Errorf("the declared create refused: %v", err)
}
for _, refused := range []string{"$JS.API.STREAM.UPDATE.KV_a_b", "$JS.API.STREAM.CREATE.KV_a_c", "$JS.API.STREAM.CREATE.*"} {
if CheckWriters(g, []string{refused}) == nil {
t.Errorf("%s was not refused", refused)
}
}
}
+1 -4
View File
@@ -213,11 +213,8 @@ func TestNoPathDeletesALogOrABucket(t *testing.T) {
// Only the writers table names it, as a subject no one but the controller may publish.
// And the operator's release 6 hand path, whose object store purges a replaced object's chunks in a node's
// object bucket (novox/hq ADR 0539 clause 10): a grant, never a call of this repository.
// And the grants part's own object bucket of sealed user lists, whose put purges a replaced object's chunks
// (novox/hq ADR 0547 clause 7): a grant, never a call of this repository.
if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") &&
!strings.HasSuffix(filepath.ToSlash(path), "internal/broker/composed.go") &&
!strings.HasSuffix(filepath.ToSlash(path), "internal/broker/userlist_switch.go") {
!strings.HasSuffix(filepath.ToSlash(path), "internal/broker/composed.go") {
found = append(found, path+": "+lit)
}
}
-11
View File
@@ -72,17 +72,6 @@ func TestAnAccountMayReadItsOwnMembershipAndNoOthers(t *testing.T) {
has(t, perms.Subscribe, "mesh.assignment.anchor.postgres")
has(t, perms.Publish, "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.postgres")
hasNot(t, perms.Subscribe, "mesh.assignment.>")
hasNot(t, perms.Publish, "mesh.assignment.*.*")
// The grants part alone publishes memberships (ADR 0547 §5), and defines no stream.
grants, err := PermissionsFor(Principal{Kind: KindModule, Node: "anchor", Module: GrantsModule, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, grants.Publish, "mesh.assignment.*.*")
has(t, grants.Publish, "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.*.*")
hasNot(t, perms.Publish, "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.*.*")
hasNot(t, grants.Publish, "$JS.API.STREAM.CREATE.>")
hasNot(t, grants.Publish, "mesh.assignment.>")
}
// The runtime arriving on a machine changes nothing about what each module is issued (to-be 38 WP2):
+3 -2
View File
@@ -52,8 +52,9 @@ func TestAModuleStreamOnARealBus(t *testing.T) {
t.Fatalf("publishing %s's mail: %v", account, err)
}
}
if got := pendingOf(js, "MAIL", c.Name, 1); got != 1 {
t.Fatalf("the reader granted `mesh` has %d pending, want 1", got)
time.Sleep(50 * time.Millisecond)
if info, _ := js.js.ConsumerInfo("MAIL", c.Name); info.NumPending != 1 {
t.Fatalf("the reader granted `mesh` has %d pending, want 1", info.NumPending)
}
narrowed := c
narrowed.Filters = []string{"mesh.stream.mesh-mail.event.mail.mesh.message.changed"}
-42
View File
@@ -214,17 +214,9 @@ type Principal struct {
// StatesItsKeyToTheVault is a machine principal whose node-engine may ask the vault's `node-key` in its own name
// (issue 786, novox/hq ADR 0372): set while the vault's holder serves it. That one verb, and nothing else new.
StatesItsKeyToTheVault bool
// FollowsGrantsUserList is the machine principal of the node holding mesh-broker: its node-engine reads the grants
// part's sealed user list, one key and an object by name (novox/hq ADR 0547 clause 7, GrantsUserListFollows).
FollowsGrantsUserList bool
// BoundBuckets are the providers' buckets of entries, `<module>_bound`, a provisioner client reads its own
// machine's keys of (mesh-sdk go/provider, issue 679): every bucket named bound of a module the mesh places.
BoundBuckets []string
// CreatesStreams are the streams the grants part's principal alone may create (novox/hq ADR 0547 §5; the operator,
// 2026-10-26 08:54): the stream of every bucket, log and stream a manifest of a module placed anywhere declares,
// by name. Created only, never updated or deleted: a new module state is on the bus when the grants part hears its
// manifest, not at the controller's next start (issue 1790).
CreatesStreams []string
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
// and never appears here: this file is written to a node's disk and read by a server, and a
@@ -916,11 +908,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
if p.StatesItsKeyToTheVault {
pub = append(pub, "mesh.seat."+VaultNodeKey.Seat+"."+CallKind+"."+VaultNodeKey.Verb+"."+CallerToken(p.Username()))
}
// And, on the node holding mesh-broker, the grants part's sealed user list it follows from take-over (novox/hq ADR
// 0547 clause 7): one key, and an object by its name.
if p.FollowsGrantsUserList {
pub = append(pub, GrantsUserListFollows()...)
}
case KindModule:
// 0. The watchman alone hears who connects and disconnects, and asks who is connected (ADR 0431 §2).
@@ -928,17 +915,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
sub = append(sub, BusConnectSubject, BusDisconnectSubject)
pub = append(pub, BusConnzSubject)
}
// The grants part alone publishes the modules' memberships into the assignments stream (novox/hq ADR 0547 §5;
// the operator, 2026-10-25 22:31): one per subject, `mesh.assignment.<node>.<module>`, reading the last one of each
// first so it publishes only what changed, and no stream it defines.
if p.Module == GrantsModule {
pub = append(pub, "mesh.assignment.*.*", "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment.*.*")
// And creating each stream a manifest declares, by its own name (the operator, 2026-10-26 08:54; issue
// 1790): the create and the read of that one stream, never an update, a delete or a stream not declared.
pub = append(pub, CreateStreamGrants(p.CreatesStreams)...)
// And its own object bucket, where it writes the user list sealed to the bus node (ADR 0547 clause 7).
pub = append(pub, GrantsUserListWrites()...)
}
// The view alone reads the build seats' workers for the backlog of asks (issue 1887 A8a).
if p.Module == ViewModule {
pub = append(pub, BuildBacklogReads()...)
@@ -1605,24 +1581,6 @@ func ComposeAccounts(principals []Principal) (string, error) {
// from them. No other principal is granted the subjects below, and nothing is a wildcard over them.
const WatchmanModule = "mesh-delivery-watchman"
// CreateStreamGrants is what creating each named stream needs, and nothing wider (novox/hq ADR 0547 §5; the operator,
// 2026-10-26 08:54): its create, and its information to read back what it holds. Each by its own name: a name that
// is not one plain subject token is left out, since a wildcard there would be every stream.
func CreateStreamGrants(streams []string) []string {
var out []string
for _, s := range streams {
if !safeSubject.MatchString(s) {
continue
}
out = append(out, "$JS.API.STREAM.CREATE."+s, "$JS.API.STREAM.INFO."+s)
}
return out
}
// GrantsModule is the one module whose principal publishes the modules' memberships (novox/hq ADR 0547 §5): the grants
// part, which composes them from the manifests and placement. It defines no stream and is granted no other subject here.
const GrantsModule = "mesh-grants"
// The bus's own events, imported into MESH from the default system account `$SYS` under subjects of MESH's own
// (novox/hq ADR 0431 §1, issue 1147): the server says an account's connections opening and closing only to the system
// account, and the list of an account's connections only to a request made there. `$SYS` holds no user, so nobody
+2 -20
View File
@@ -24,9 +24,8 @@ import (
// subjects cover it, so what travels there — a code typed by the operator — is never persisted. A kinded
// holder asks with its own kind; the modules that watch the seat answer.
//
// And two reads: **a holder's records**, a bucket the seat names, read by each user under its own name only
// (`$KV.<bucket>.<module>.>`), so an asker reads the state of its own asks and no other asker's; and on a seat pulled by
// kind, **what its workers gave up on**, the copies DEAD_LETTERS keeps of the seat's stream (DeadLetterReads, issue 734).
// And one read: **a holder's records**, a bucket the seat names, read by each user under its own name only
// (`$KV.<bucket>.<module>.>`), so an asker reads the state of its own asks and no other asker's.
// BuildAskMaxDeliver bounds the deliveries of a build seat's ask (issue 1907): mesh-sdk delivery.AskMaxDeliver, written
// here as the broker mirrors the catalogue rather than importing it.
@@ -130,9 +129,6 @@ func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic {
}
t.Records = append(t.Records, "$JS.API.DIRECT.GET.KV_"+b+".$KV."+b+"."+module+".>")
}
if s.PulledByKind && len(s.Accepts) > 0 && safeSubject.MatchString(s.Name) {
t.Records = append(t.Records, DeadLetterReads(seatStreamName(s.Name))...)
}
}
for _, w := range watches {
if w.Kinded {
@@ -155,20 +151,6 @@ func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic {
return t
}
// DeadLetterReads is what a user of a seat pulled by kind is granted to hear which of the asks it put the seat's workers
// gave up on (novox/hq issue 734): an ordered consumer on DEAD_LETTERS filtered to the copies the controller keeps of that
// seat's stream (DeadLetterSubject(stream, *)), pulled, and its deletion. A copy carries the ask's own message id
// (Mesh-Dead-Msg-Id), so the asker ends the step it waits for instead of waiting for ever; it reads the seat's dead
// letters and no other stream's, and changes none.
func DeadLetterReads(stream string) []string {
return []string{
"$JS.API.STREAM.INFO." + DeadLettersStream,
"$JS.API.CONSUMER.CREATE." + DeadLettersStream + ".*." + DeadLetterSubject(stream, ">"),
"$JS.API.CONSUMER.MSG.NEXT." + DeadLettersStream + ".*",
"$JS.API.CONSUMER.DELETE." + DeadLettersStream + ".*",
}
}
// seatTrafficGrants is the bus permissions seat traffic needs: the subjects themselves, and the JetStream API a
// worker is pulled and acknowledged through and a record is read through.
func seatTrafficGrants(t SeatTraffic) (pub, sub []string) {
-13
View File
@@ -403,19 +403,6 @@ func TestTheBuilderMayAskABuildSeatAndASessionMayNot(t *testing.T) {
t.Errorf("the builder may not publish %s", s)
}
}
// Issue 734: the asker hears which of its asks the seat's workers gave up on, from DEAD_LETTERS, and no other stream's.
for subject, want := range map[string]bool{
"$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x1.mesh.events.dead.SEAT_NODE_BUILD_GO.>": true,
"$JS.API.CONSUMER.DELETE.DEAD_LETTERS.x1": true,
"$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTERS.x1": true,
"$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x1.mesh.events.dead.EVENTS.>": false,
"$JS.API.STREAM.PURGE.DEAD_LETTERS": false,
"$JS.FC.DEAD_LETTERS.x1": false,
} {
if got := allowed(builder.Publish, subject); got != want {
t.Errorf("the builder may publish %s: %v, want %v", subject, got, want)
}
}
session := perms(t, Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"mesh-controller.status"}})
if allowed(session.Publish, "mesh.seat.node-build-go.accept.delivery") {
t.Error("a session's account may put an ask")
-55
View File
@@ -1,55 +0,0 @@
package broker
// **The grants part's user list, sealed to the bus node** (novox/hq ADR 0547 clause 7, issue 1945): the operator's
// second exception to the frozen controller (the first, memberships, 2026-10-25 22:31). The controller's copy of the bus
// composer grants what mesh-sdk's copy grants, byte for byte, and nothing else: the node-engine of the node holding
// mesh-broker reads the grants part's one key `user-list.sealed`, watched, and an object of its bucket by name; the
// grants part writes that bucket. Without it the bus node could never read the first list the grants part writes, since
// until take-over the list the bus loads is this one (mesh-sdk go/buscompose/userlist_switch.go keeps the other copy).
// The grants part's user list on the bus (mesh-sdk go/delivery userlist.go names them).
const (
GrantsUserListBucket = "mesh-grants_user-list"
GrantsUserListKey = "sealed"
GrantsUserListObjects = "mesh-grants-user-list"
GrantsUserListObjectStream = "OBJ_" + GrantsUserListObjects
)
// GrantsUserListFollows is what the node-engine of the node holding mesh-broker may publish to follow the grants part's
// list: the one key read and watched, and an object of the bucket by its name (as ADR 0486 §2 reads a declaration's
// files: never the object stream's info, which lists every name, never a message by sequence).
func GrantsUserListFollows() []string {
kv := "KV_" + GrantsUserListBucket
key := "$KV." + GrantsUserListBucket + "." + GrantsUserListKey
obj, s := GrantsUserListObjects, GrantsUserListObjectStream
return []string{
"$JS.API.DIRECT.GET." + kv + "." + key,
"$JS.API.CONSUMER.CREATE." + kv + ".*." + key,
"$JS.API.CONSUMER.DELETE." + kv + ".*",
"$JS.API.STREAM.INFO." + kv,
"$JS.FC." + kv + ".>",
"$JS.API.DIRECT.GET." + s + ".$O." + obj + ".M.*",
"$JS.API.CONSUMER.CREATE." + s + ".*.$O." + obj + ".C.*",
"$JS.API.CONSUMER.DELETE." + s + ".*",
"$JS.FC." + s + ".>",
}
}
// GrantsUserListWrites is what the grants part may publish to keep its object bucket.
func GrantsUserListWrites() []string {
obj, s := GrantsUserListObjects, GrantsUserListObjectStream
return []string{
"$JS.API.STREAM.CREATE." + s,
"$JS.API.STREAM.INFO." + s,
"$JS.API.STREAM.PURGE." + s,
"$JS.API.DIRECT.GET." + s + ".>",
"$JS.API.STREAM.MSG.GET." + s,
"$O." + obj + ".>",
}
}
// grantsOwnObjects is the grants part writing its own object bucket, and nothing of another.
func grantsOwnObjects(p Principal, pattern string) bool {
return p.Kind == KindModule && p.Module == GrantsModule &&
(pattern == "$O."+GrantsUserListObjects+".>" || pattern == "$JS.API.STREAM.CREATE."+GrantsUserListObjectStream)
}
+1 -37
View File
@@ -147,19 +147,14 @@ func Users(r Records) ([]Principal, error) {
for _, node := range sortedCopy(r.Nodes) {
witness := false
var checks []string
bus := false
for _, d := range r.Assigned[node] {
if d.Module == controllerModule {
witness = true
}
if d.SnapshotsTheBus {
bus = true
}
checks = append(checks, d.Checks...)
}
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks,
MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey),
FollowsGrantsUserList: bus})
MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey)})
// **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one
// per machine that runs it, delivered to the module running it.
for _, d := range r.Assigned[node] {
@@ -212,11 +207,6 @@ func Users(r Records) ([]Principal, error) {
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
Logs: logNames(d.Logs), Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues,
})
// **The grants part creates what a manifest declares** (novox/hq ADR 0547 §5; the operator, 2026-10-26
// 08:54): every placed module's buckets, logs and streams, each by its stream's name.
if d.Module == GrantsModule {
out[len(out)-1].CreatesStreams = declaredStreamsOf(r)
}
// **The vault's holder asks the desks and the issuing providers itself** (issue 723), under the
// account its own runtime runs on (ADR 0259 §8's runs-as): the one module granted it.
if holdsSeat(d, VaultSeat) {
@@ -428,29 +418,3 @@ func boundBucketsOf(r Records) []string {
func runnerServes(d Declared) bool {
return d.Module == RuntimeModule || d.NoAccount || d.Loads || len(d.Serves) > 0 || len(d.Holds) > 0 || len(d.Waits) > 0 || len(d.Checks) > 0
}
// declaredStreamsOf is the stream of every bucket, log and stream a module placed anywhere declares, once each, in name
// order (novox/hq ADR 0547 §5, issue 1790): `KV_<module>_<state>`, `LOG_<module>_<log>` and each module stream's own
// name. What the grants part may create, and nothing beside.
func declaredStreamsOf(r Records) []string {
seen := map[string]bool{}
for _, mods := range r.Assigned {
for _, d := range mods {
for _, b := range d.State {
seen["KV_"+b.Bucket()] = true
}
for _, l := range d.Logs {
seen[l.Stream()] = true
}
for _, s := range d.Streams {
seen[s.Name] = true
}
}
}
out := make([]string, 0, len(seen))
for s := range seen {
out = append(out, s)
}
sort.Strings(out)
return out
}
+2 -20
View File
@@ -37,20 +37,6 @@ type WriterRow struct {
// isController, and the other writers' tests, are who a row's writer is as a principal.
func isController(p Principal, _ string) bool { return p.Kind == KindController }
// createsDeclared is the controller, or the grants part creating one stream a manifest declares, by its own name: the
// create alone, never an update or a delete (novox/hq ADR 0547 §5; the operator, 2026-10-26 08:54).
func createsDeclared(p Principal, pattern string) bool {
if isController(p, pattern) {
return true
}
name, ok := strings.CutPrefix(pattern, "$JS.API.STREAM.CREATE.")
if ok && p.Kind == KindModule && p.Module == GrantsModule && name == GrantsUserListObjectStream {
return true
}
return ok && p.Kind == KindModule && p.Module == GrantsModule && slices.Contains(p.CreatesStreams, name) &&
safeSubject.MatchString(name)
}
// ownMachine is a node writing a subject whose third token is its own name, and no wildcard there.
func ownMachine(p Principal, pattern string) bool {
tokens := strings.Split(pattern, ".")
@@ -184,8 +170,7 @@ var WritersTable = []WriterRow{
// bucket's stream (ADR 0201), which defines nothing the mesh keeps.
Subjects: []string{"$JS.API.STREAM.CREATE.>", "$JS.API.STREAM.UPDATE.>", "$JS.API.STREAM.DELETE.>",
"$JS.API.CONSUMER.DURABLE.CREATE.>"},
Writes: createsDeclared,
Shared: "the grants part creates a stream a manifest declares, by its name, and never updates or deletes one (novox/hq ADR 0547 §5; the operator, 2026-10-26 08:54)"},
Writes: isController},
{State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state",
Others: "the controller asks",
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
@@ -252,10 +237,7 @@ var WritersTable = []WriterRow{
KeptIn: "the object bucket " + DeclarationFilesBucket + "-<node> of each node, an object removed a day after the " +
"node's latest declaration no longer names it",
Others: "a node-engine reads an object of its own bucket by its name and may neither list the bucket nor write it",
Subjects: []string{"$O.*.>"}, Writes: func(p Principal, pattern string) bool {
return isControllerOrTheHandPath(p, pattern) || grantsOwnObjects(p, pattern)
},
Shared: "the grants part writes its own object bucket " + GrantsUserListObjects + ", the user list sealed to the bus node, and no other (novox/hq ADR 0547 clause 7)"},
Subjects: []string{"$O.*.>"}, Writes: isControllerOrTheHandPath},
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
+3 -12
View File
@@ -67,23 +67,14 @@ func (i *Inventory) HighestEpoch(ctx context.Context) (uint64, error) {
}
// TookEpoch records an epoch taken, and ends every earlier one still open as found expired: the lease
// was free to take, so whoever held it last neither holds it nor said it let go. givenBack is the epoch the
// bus says gave the key back (the lease's own delete marker names it, novox/hq issue 906, ADR 0558 §3): that
// epoch, if still open, is ended as released — its holder gave the key back and had not yet recorded it — and
// every other as expired; zero is none. Answers the epochs it ended as expired, newest first.
func (i *Inventory) TookEpoch(ctx context.Context, e Epoch, givenBack uint64) ([]Epoch, error) {
// was free to take, so whoever held it last neither holds it nor said it let go. Answers the epochs it
// ended that way, newest first.
func (i *Inventory) TookEpoch(ctx context.Context, e Epoch) ([]Epoch, error) {
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
if givenBack != 0 && givenBack < e.Epoch {
if _, err := tx.Exec(ctx,
`update controller_epoch set ended = now(), how = $2 where epoch = $1 and ended is null`,
int64(givenBack), EpochReleased); err != nil {
return nil, err
}
}
rows, err := tx.Query(ctx,
`update controller_epoch set ended = now(), how = $2
where ended is null and epoch < $1
+2 -3
View File
@@ -127,10 +127,9 @@ var (
// begin.
var PlacementKinds = []string{"dir", "access", "port", "seat", "machine", "backup"}
// credentialForms are the forms a provider's offer may declare for its credential (issue 768); "none" is an offer with
// no credential, which the release's catalogue states on every such offer (issue 774).
// credentialForms are the forms a provider's offer may declare for its credential (issue 768).
var credentialForms = []string{catalogue.CredentialFormShared, catalogue.CredentialFormPair,
catalogue.CredentialFormIssued, catalogue.CredentialFormGiven, catalogue.CredentialFormNone}
catalogue.CredentialFormIssued, catalogue.CredentialFormGiven}
var (
digestShape = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+2 -4
View File
@@ -316,16 +316,14 @@ func TestAPlanNamesTheCredentialFormOfEachProvidersOffer(t *testing.T) {
p := aPlan()
p.Provisions = []ProvisionedCredential{
{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres", CredentialForm: "pair"},
{Provision: "sonarr-api", Provider: "ace", ProviderModule: "sonarr", CredentialForm: "shared"},
{Provision: "artifact-store", Provider: "novox", ProviderModule: "distribution", CredentialForm: "none"}}
{Provision: "sonarr-api", Provider: "ace", ProviderModule: "sonarr", CredentialForm: "shared"}}
values, err := AssignmentValues(assigned, map[string]Provisioning{"g14.dunst": p})
if err != nil {
t.Fatal(err)
}
if want := `{"provision":"postgres-database","provider":"novox","provider-module":"postgres",` +
`"credential-form":"pair"},{"provision":"sonarr-api","provider":"ace","provider-module":"sonarr",` +
`"credential-form":"shared"},{"provision":"artifact-store","provider":"novox","provider-module":"distribution",` +
`"credential-form":"none"}`; !strings.Contains(string(values["g14.dunst"]), want) {
`"credential-form":"shared"}`; !strings.Contains(string(values["g14.dunst"]), want) {
t.Errorf("the plan's value is %s, want it to carry %s", values["g14.dunst"], want)
}
bad := aPlan()
+9 -71
View File
@@ -34,11 +34,9 @@ import (
"encoding/json"
"errors"
"fmt"
"strconv"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
@@ -117,7 +115,6 @@ type Options struct {
// Lease is one instance's hold, or its wait for one.
type Lease struct {
kv jetstream.KeyValue
js jetstream.JetStream
stream jetstream.Stream
ttl time.Duration
o Options
@@ -142,9 +139,6 @@ type Lease struct {
lostWhy error
lost chan struct{}
taken time.Time
// givenBack is the epoch whose holder gave back the key this instance took — the epoch its delete marker
// names on the bus — and zero when the key was left to expire, or freed by anything else (novox/hq issue 906).
givenBack uint64
// update writes the key at a revision: the bucket's own, or a test's bus that does not answer.
update func(ctx context.Context, key string, value []byte, last uint64) (uint64, error)
}
@@ -190,7 +184,7 @@ func Open(ctx context.Context, js jetstream.JetStream, bucket string, o Options)
return nil, fmt.Errorf("a lease renewed every %s with a margin of %s does not fit a key that lives %s",
o.RenewEvery, o.Margin, status.TTL())
}
return &Lease{kv: kv, js: js, stream: stream, ttl: status.TTL(), o: o, lost: make(chan struct{}), update: kv.Update}, nil
return &Lease{kv: kv, stream: stream, ttl: status.TTL(), o: o, lost: make(chan struct{}), update: kv.Update}, nil
}
// TTL is how long the key lives unrenewed.
@@ -277,7 +271,6 @@ func (l *Lease) TryTake(ctx context.Context) (uint64, error) {
} else if found {
return 0, &heldBy{h}
}
givenBack := l.wasGivenBack(ctx)
if err := l.aboveTheFloor(ctx); err != nil {
return 0, err
}
@@ -314,51 +307,13 @@ func (l *Lease) TryTake(ctx context.Context) (uint64, error) {
"back to be taken again", revision, floor)
}
l.mu.Lock()
l.held, l.epoch, l.revision, l.taken, l.givenBack = true, revision, revision, now, givenBack
l.held, l.epoch, l.revision, l.taken = true, revision, revision, now
l.renewedAt(anchor)
l.mu.Unlock()
l.o.Say("took the controller lease at epoch %d (%s)", revision, h.Instance)
return revision, nil
}
// ReleasedHeader is the header of the delete marker a holder's Release writes, naming the epoch it gave back:
// the one mark that says a handover. A delete marker without it — a command's own lease given back, a take
// given back over the floor — names no epoch, so it says nothing about the holder before (novox/hq issue 906).
const ReleasedHeader = "Mesh-Lease-Released"
// wasGivenBack is the epoch the free key was given back by: the epoch its last message on the bus — the delete
// marker a holder's Release writes — names. Zero when the key expired (the bucket's age removed every message),
// when the marker names no epoch, and when it cannot be read: a takeover nobody can prove was a handover is
// said as one that was not (novox/hq issue 906, ADR 0558 §3).
func (l *Lease) wasGivenBack(ctx context.Context) uint64 {
if l.stream == nil {
return 0
}
last, err := l.stream.GetLastMsgForSubject(ctx, l.subject())
if err != nil || last == nil {
return 0
}
if op := last.Header.Get("KV-Operation"); op != "DEL" && op != "PURGE" {
return 0
}
epoch, err := strconv.ParseUint(last.Header.Get(ReleasedHeader), 10, 64)
if err != nil {
return 0
}
return epoch
}
// subject is the lease key's subject in its bucket's stream.
func (l *Lease) subject() string { return "$KV." + l.kv.Bucket() + "." + Key }
// GivenBack is the epoch whose holder gave back the key this instance took, zero when it was not given back:
// what the takeover was, read from the bus (novox/hq issue 906).
func (l *Lease) GivenBack() uint64 {
l.mu.Lock()
defer l.mu.Unlock()
return l.givenBack
}
// floor is the highest epoch issued, zero without a floor.
func (l *Lease) floor(ctx context.Context) (uint64, error) {
if l.o.Floor == nil {
@@ -633,11 +588,9 @@ func (l *Lease) lose(why error) error {
return l.lostWhy
}
// Release gives the lease back, so the next candidate takes it at once rather than after its age, and answers
// the epoch it gave back — zero when it gave nothing back: it held none, had lost it, or the bus would not
// delete the key. Only the key this instance last wrote is deleted: one that moved is somebody else's. The
// gate closes before the key goes, so nothing this instance starts after overlaps its successor.
func (l *Lease) Release(ctx context.Context) uint64 {
// Release gives the lease back, so the next candidate takes it at once rather than after its age.
// Only the key this instance last wrote is deleted: one that moved is somebody else's.
func (l *Lease) Release(ctx context.Context) {
l.mu.Lock()
held, revision, epoch := l.held && l.lostWhy == nil, l.revision, l.epoch
l.held = false
@@ -647,11 +600,11 @@ func (l *Lease) Release(ctx context.Context) uint64 {
}
l.mu.Unlock()
if !held {
return 0
return
}
releasing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
err := l.giveBack(releasing, epoch, revision)
err := l.kv.Delete(releasing, Key, jetstream.LastRevision(revision))
if err != nil && isWrongLast(err) {
// A renewal was in flight as this began and landed first: the key moved, and is still this
// instance's if it still names it at this epoch. Deleted at the revision it has now.
@@ -659,29 +612,14 @@ func (l *Lease) Release(ctx context.Context) uint64 {
var h Holder
if json.Unmarshal(entry.Value(), &h) == nil && h.Instance == l.o.Holder.Instance &&
(h.Epoch == epoch || h.Epoch == 0 && entry.Revision() == epoch) {
err = l.giveBack(releasing, epoch, entry.Revision())
err = l.kv.Delete(releasing, Key, jetstream.LastRevision(entry.Revision()))
}
}
}
if err != nil {
l.o.Say("the controller lease (epoch %d) could not be given back, so the next controller waits for "+
"it to expire: %v", epoch, err)
return 0
return
}
l.o.Say("gave the controller lease back (epoch %d)", epoch)
return epoch
}
// giveBack deletes the key at the revision this instance last wrote, as the bucket's own delete does — a
// marker with KV-Operation DEL, compare-and-set on the key's last revision — and names the epoch given back in
// the marker, so the next holder can tell this handover from any other delete (ReleasedHeader).
func (l *Lease) giveBack(ctx context.Context, epoch, revision uint64) error {
if l.js == nil {
return l.kv.Delete(ctx, Key, jetstream.LastRevision(revision))
}
msg := nats.NewMsg(l.subject())
msg.Header.Set("KV-Operation", "DEL")
msg.Header.Set(ReleasedHeader, strconv.FormatUint(epoch, 10))
_, err := l.js.PublishMsg(ctx, msg, jetstream.WithExpectLastSequencePerSubject(revision))
return err
}
+3 -17
View File
@@ -133,12 +133,7 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
// a gives it back: b takes it at once, not after the key's age, at a higher epoch.
stop()
handedOver := time.Now()
if given := a.Release(t.Context()); given != epochA {
t.Fatalf("a's release answers epoch %d given back, want %d", given, epochA)
}
if again := a.Release(t.Context()); again != 0 {
t.Fatalf("a second release gives epoch %d back again", again)
}
a.Release(t.Context())
select {
case epochB := <-took:
if epochB <= epochA {
@@ -147,9 +142,6 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
if waited := time.Since(handedOver); waited > testTTL {
t.Fatalf("b took the lease %s after a gave it back: it waited out the key's age", waited)
}
if given := b.GivenBack(); given != epochA {
t.Fatalf("b reads from the bus that epoch %d gave the key back, want a's %d (novox/hq issue 906)", given, epochA)
}
case <-time.After(2 * testTTL):
t.Fatal("b never took the lease a gave back")
}
@@ -190,9 +182,6 @@ func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
if epochB <= epochA {
t.Fatalf("b took epoch %d after a's %d", epochB, epochA)
}
if given := b.GivenBack(); given != 0 {
t.Fatalf("b reads a key that expired as given back by epoch %d (novox/hq issue 906)", given)
}
// And a, the moment it tries to renew, knows it lost: the key moved.
if err := a.Renew(t.Context()); err == nil {
t.Fatal("a renewed a lease b holds")
@@ -237,11 +226,8 @@ func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
if _, err := a.Epoch(); !errors.Is(err, ErrNotHeld) {
t.Fatalf("a acts after its renewal was refused: %v", err)
}
// And giving back a lease it lost deletes nothing of the one who holds it now, and answers no epoch given
// back, so none is recorded released (novox/hq issue 906).
if given := a.Release(t.Context()); given != 0 {
t.Fatalf("a says it gave back epoch %d of a lease it lost", given)
}
// And giving back a lease it lost deletes nothing of the one who holds it now.
a.Release(t.Context())
if h, found, err := Current(t.Context(), kv); err != nil || !found || h.Instance != "intruder" {
t.Fatalf("after a gave back what it lost, the key holds %+v (%v, %v)", h, found, err)
}
@@ -1,8 +1,6 @@
package ownmanifest
import (
"path/filepath"
"slices"
"testing"
. "git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -96,21 +94,8 @@ func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) {
// 2 and 4): a Go bundle of the module's own Go module stamped for alpine, unpacked by an archive resource, mounted
// read-only at /opt/mesh, called through args, and named under restart-on so a new program never leaves the old
// one running. The container's name, network and logging are what the jail and the routes rely on.
//
// And its routes are the seat mesh-provisioner's plans alone (issue 774, research 072 row 34): it reads them, and
// receives no file of contributions; no module of this repository does.
func TestRouteProxyRunsItsMountedProgramOnTheTrustImage(t *testing.T) {
m := catalogueManifest(t, "route-proxy")
if !slices.Contains(m.Reads, "mesh-provisioner.plans") {
t.Errorf("route-proxy does not read mesh-provisioner.plans, where its routes are: %v", m.Reads)
}
dirs, _ := filepath.Glob(filepath.Join("..", "..", "modules", "*"))
for _, dir := range dirs {
if r := catalogueManifest(t, filepath.Base(dir)); len(r.Receives) > 0 {
t.Errorf("%s receives %v: a route, like every provision here, is served from the seat's plans, never a "+
"received file", r.Module, r.Receives)
}
}
var bundle *Artifact
for i, a := range m.Build.Artifacts {
if a.Name == "server" {
-1
View File
@@ -82,7 +82,6 @@
"dead-letters",
"root-free",
"judge-directories",
"placement-export",
"data",
"build",
"artifacts",
+14 -33
View File
@@ -5,31 +5,26 @@ The shipping form of the reference reverse proxy (novox/hq ADR 0007, novox/hq 08
## What it is
**A route is a provision** (issue 774, research 072 row 34; the operator, 2026-10-31: "there is only 1 path, the other
is simply deprecated/invalid"). A module that must be reachable declares `requires: ["route"]` and contributes the
name it wants and the port it listens on; this module provides `route`. The controller composes each consumer's plan,
and the seat `mesh-provisioner` keeps it in its state `plans`: the plan's entry for the route names this proxy as
its provider and carries what the consumer contributes, composed (`given`: its names joined to the node's domains, its
endpoint's port, its policy), and where the consumer's machine is on the private network (`consumer-at`). The proxy
follows the plans on the bus and forwards by the `Host` header of each request to `http://<consumer-at>:<port>`, or
over loopback for a consumer on its own machine.
**A route is a grant.** A module that must be reachable declares `requires: ["route"]` and
contributes the name it wants and the port it listens on; this module provides `route`, receives
every consumer that asked as a file, and forwards by the `Host` header of each request to
`http://<at>:<port>` — where `at` is where the mesh says that consumer's machine is (empty means
this machine, reached over loopback).
The controller does not write the proxy's configuration. It composes the facts into the plans and the proxy turns them
into whatever it runs. That boundary is why swapping this proxy for another costs nothing to the modules that publish
through it. **There is no received file and no second source:** a route no plan names is not served, and a module
declaring `receives.route` is refused by the catalogue check (mesh-catalog `checks/credentialform`), but for the
migration scaffolding still listed there.
The control plane does not write the proxy's configuration. It delivers the facts — the
contributions file at `receives.route` — and the proxy turns them into whatever it runs. That
boundary is why swapping this proxy for another costs nothing to the modules that publish through
it.
| field | value | why |
|---|---|---|
| `provides` | `route` (scope `mesh`, answered by the helper) | a module on any node may require it |
| `provides` | `route` (scope `mesh`) | a module on any node may require it |
| `serves.route` | `{}` | a route hands back a name, not a credential — the consumer already knows the name it asked for |
| `reads` | `mesh-provisioner.plans`, `mesh-provisioner.provisions` | the plans are where the routes are; the helper reads both |
| `receives.route` | `…/routes/mesh.json` | where the mesh writes every contribution; the proxy reads it as `$ROUTES` and re-reads on change |
| `listens` | `80` and `443`, both `from: anywhere` | the one machine with a public opening; the firewall opens these for free (ADR 0045) |
Until the bus answers, the proxy routes nothing and says so on its log; it refuses to start without its broker
credential (`MESH_BROKER_FILE`). Who is inside the private network, and so may be served an internal-only name, is
still read from its membership (novox/hq ADR 0167).
No provisioner: the proxy neither mints a credential for anybody nor answers a provision other
than `route`. It reads the file the mesh writes, and its own broker credential is for its tools.
## Which issuer: Let's Encrypt, stated here (novox/hq ADR 0226)
@@ -82,7 +77,7 @@ traffic.
| env | default | meaning |
|---|---|---|
| `MESH_BROKER_FILE` | *(required)* | the proxy's broker credential: the plans it routes from, and its membership |
| `ROUTES` | `/routes/mesh.json` | the contributions file the mesh writes (`receives.route`) |
| `LISTEN` | `:80` | HTTP, and the ACME HTTP-01 challenge |
| `TLS_LISTEN` | `:443` | HTTPS; unset to serve plain HTTP only |
| `ACME_CACHE` | `/acme` | where issued certificates persist; required when `TLS_LISTEN` is set, so a restart does not re-order |
@@ -95,17 +90,3 @@ A small Go program beside the proxy (`cmd/route-provider`) runs the SDK's provid
and the offer says `"answered-by": "helper"`: a handed consumer (gitea's origins file) is filled by its machine's client
from this module's entry, which carries the public names the controller composed for it. The module is **not** marked
moved: its own files, `acme.env` and the broker secret, are composed as before, and the proxy itself is unchanged.
## What it says on the bus, and its tools (issue 774)
The proxy keeps two states, announced readable (`conn.Keeps`, novox/hq ADR 0495), each keyed `<node>.<name>`:
| state | one key per | holds | written when |
|---|---|---|---|
| `routes` | served name | the rules it is served by (path, module, node, target, refusal, redirect, authentication); its certificate (authority, issuer, not-after, last renewal, outcome, or why it failed); the last upstream answer (status, time, error, path) | a route arrives, changes or leaves; an authority stores a certificate; a handshake fails to find one or hands out one not yet recorded; an upstream answer differs from the last one written, or the same one is a minute old |
| `unrouted` | name answered "no route for", in a request or a TLS handshake, under a domain the proxy serves | when it was last asked, how often since the start, whether it is an internal-only name asked from outside; never who asked (the journal keeps that, for fail2ban) | first asked, at most five new names a minute, then at most once a minute; deleted when the name is served; kept a week. Every other refusal is only counted, in one key per proxy (`<node>._other`: the last minute's count and the total), written once a minute. A refused write is dropped, never retried |
Nothing is written on a timer. The provider helper's binary serves the module's tools from those states, one call each:
`route_proxy_routes` (every served name with its target, certificate expiry and last answer), `route_proxy_route`
(one name in full, or that it is not served and when it was last asked for) and `route_proxy_unrouted` (the names
refused, newest first).
+3 -11
View File
@@ -1,6 +1,6 @@
// route-provider: route-proxy's Go half, one binary the node's runtime launches and speaks MCP to over stdio
// through the Go SDK. It runs the SDK's provider helper that answers the consumers of `route` (issue 792), and serves
// the module's tools, which read what the proxy keeps on the bus (issue 774, tools.go).
// through the Go SDK. It serves no tools; it runs the SDK's provider helper that answers the consumers of `route`
// (issue 792). The TypeScript adapter beside it keeps writing the proxy's files.
//
// stdout is the MCP channel, so everything this module says, it says on stderr.
package main
@@ -9,12 +9,9 @@ import (
"context"
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/provider"
"route-proxy/internal/routestate"
)
func say(format string, args ...any) {
@@ -28,17 +25,12 @@ func main() {
} else {
p := newProvider(node, provider.StateBus(ModuleName), say)
go func() {
// The helper's watches go through the runtime's channel, which Serve holds: one asked before it does is
// refused ("before Serve"), Run returns and the helper stays stopped until the process restarts.
<-stdio.Served()
if err := p.Run(context.Background()); err != nil {
say("the provider helper stopped: %v", err)
}
}()
}
tools := front{routes: stdio.State(routestate.RoutesState), unrouted: stdio.State(routestate.UnroutedState),
now: time.Now}.tools()
if err := stdio.Serve("", tools); err != nil {
if err := stdio.Serve("", nil); err != nil {
say("%v", err)
os.Exit(1)
}
@@ -7,20 +7,18 @@ package main
// **It composes nothing.** A route's public name is the label its consumer chose joined to the node's public domain,
// which only the controller knows (the domain is placement), so the controller composes it and carries it in the
// consumer's plan entry (`names`); the helper copies it into the entry's values unchanged. There is no credential for
// a route and nothing to make. The proxy itself serves the same plans' entries (cmd/route-proxy plans.go), the one
// source of routes (issue 774, research 072 row 34).
// a route and nothing to make: the traefik files stay the TypeScript adapter's, written from the contributions the
// mesh still gives it.
import (
"git.novox.be/novox/mesh-sdk/go/provider"
"route-proxy/internal/routestate"
)
// ProvisionName is what route-proxy provides, as its manifest says, and ModuleName this module.
const (
ProvisionName = routestate.Provision
ModuleName = routestate.Module
)
// ProvisionName is what route-proxy provides, as its manifest says.
const ProvisionName = "route"
// ModuleName is this module.
const ModuleName = "route-proxy"
// newProvider is route-proxy answering its consumers over bus on node: the route, served as the manifest serves it
// (nothing of its own), with no credential, and nothing to make or retire for a consumer.
@@ -9,9 +9,7 @@ import (
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// bucketBus is the helper's bus over in-memory buckets.
@@ -46,60 +44,33 @@ func (b *bucketBus) Delete(_ context.Context, bucket, key string) error {
return nil
}
// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792),
// from the seat mesh-provisioner's plans and keys (novox/hq ADR 0570 clause 2), never the controller's assignments: a
// module not handed whose route the seat keys (grafana, issue 774) has a plan of provisions alone, and a helper that
// read only the assignments left its key asked for good, so the assigner withheld every new build of 38 such modules.
// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792):
// the entry's values are the controller's composition, copied unchanged, and no vault is asked.
func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) {
bus := &bucketBus{buckets: map[string]map[string][]byte{}}
for module, names := range map[string]map[string]string{
"gitea": {"name": "git.example.org", "internal-name": "git.novox.internal", "name-web": "git.example.org"},
"grafana": {"name": "grafana.example.org"},
} {
plan, err := provisions.EncodePlan(provisions.Plan{Node: "g14", Module: module, Plan: catalogue.Plan{
Provisions: []catalogue.PlanProvision{{Provision: ProvisionName, Provider: "novox", ProviderModule: ModuleName,
At: "10.7.0.1", From: "novox", As: "mesh_g14_" + module, Names: names}}}})
if err != nil {
t.Fatal(err)
}
key := "g14." + module
if err := bus.Put(context.Background(), provisions.PlansBucket, key, plan); err != nil {
t.Fatal(err)
}
asked, err := json.Marshal(provisions.Provision{V: provisions.Version, Provision: ProvisionName,
Consumer: provisions.End{Node: "g14", Module: module}, Provider: provisions.End{Node: "novox", Module: ModuleName},
State: provisions.Asked})
if err != nil {
t.Fatal(err)
}
if err := bus.Put(context.Background(), provisions.ProvisionsBucket, key+"."+ProvisionName, asked); err != nil {
t.Fatal(err)
}
plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{
"provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox",
"provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea",
"names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal",
"name-web": "git.example.org"}}}}})
if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil {
t.Fatal(err)
}
p := newProvider("novox", bus, t.Logf)
out, err := p.Pass(context.Background())
if err != nil {
t.Fatal(err)
}
if len(out) != 2 || out[0].Status != provider.Answered || out[1].Status != provider.Answered {
if len(out) != 1 || out[0].Status != provider.Answered {
t.Fatalf("the pass did %+v", out)
}
for key, want := range map[string]map[string]string{
"g14.gitea.route": {"name-web": "git.example.org", "name": "git.example.org", "internal-name": "git.novox.internal"},
"g14.grafana.route": {"name": "grafana.example.org"},
} {
var e provider.Entry
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)][key], &e); err != nil {
t.Fatalf("%s: %v", key, err)
}
for k, v := range want {
if e.Values[k] != v {
t.Errorf("%s says %s %v, want %s", key, k, e.Values[k], v)
}
}
if e.State != provisions.Provided || e.VaultName != "" || e.User != "" || e.Refused != "" {
t.Errorf("the entry %s is %+v", key, e)
}
var e provider.Entry
if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)]["g14.gitea.route"], &e); err != nil {
t.Fatal(err)
}
if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" ||
e.Values["internal-name"] != "git.novox.internal" || e.VaultName != "" || e.User != "" || e.Refused != "" {
t.Fatalf("the entry is %+v", e)
}
}
@@ -108,8 +79,6 @@ func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) {
// the public ingress does not change. Everything of the manifest but its build, what it provides and the state it
// keeps is pinned by this digest; changing the ingress means changing it here, on purpose. Re-pinned in R4d (novox/hq
// ADR 0480): the program is an archive mounted into the trust image, the container and its files are otherwise as before.
// Re-pinned for issue 774 (research 072 row 34): the received routes file, its directory, mount and ROUTES went, since
// the proxy serves from the seat's plans alone; the ingress, its certificates and its files are otherwise as before.
func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
@@ -126,8 +95,7 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) {
if len(provides) != 1 || provides[0].(map[string]any)["answered-by"] != "helper" {
t.Fatalf("route is not said to be answered by the helper: %v", provides)
}
// Its states and its tools are what it says on the bus, not what it runs or writes (issue 774).
for _, k := range []string{"build", "provides", "state", "reads", "tools", "replaces"} {
for _, k := range []string{"build", "provides", "state", "reads"} {
delete(m, k)
}
pinned, err := json.Marshal(m) // sorted keys, compact
@@ -135,7 +103,7 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) {
t.Fatal(err)
}
sum := sha256.Sum256(pinned)
if got := hex.EncodeToString(sum[:]); got != "e96702560003ae7f27490d27fd2353f8c6680d8e16864e15fdc0a833108bc2e5" {
if got := hex.EncodeToString(sum[:]); got != "85cb955e5a4b2d8826f50aa76433064a576e8b15b71537032eb93c7388c0ee28" {
t.Fatalf("route-proxy's own manifest changed: digest %s", got)
}
// The two acme templates are pinned the same way (issue 849): they are what the proxy's certificates are asked
@@ -1,296 +0,0 @@
package main
// route-proxy's tools (issue 774): the web front's command line. Each answers from what the proxy keeps on the bus
// (route-proxy_routes and route-proxy_unrouted, written on change by the proxy itself), in one call, for every
// machine running a proxy: which sites are served and where they go, whether their certificates are good, what the
// last request through each got back, and which names were asked for and are served nowhere.
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// The tools' names, as the manifest declares them.
const (
RoutesTool = "route_proxy_routes"
RouteTool = "route_proxy_route"
UnroutedTool = "route_proxy_unrouted"
)
// reader is one of the module's states, as stdio.KeptState reads it.
type reader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// front is what the tools read: the routes and the refusals, and the clock a certificate's days left are counted by.
type front struct {
routes, unrouted reader
now func() time.Time
}
// RouteLine is one served name in the list: where it goes, how long its certificate holds, what it last answered.
type RouteLine struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
Reach string `json:"reach"`
Targets []string `json:"targets"`
// Reachable is yes when the last answer upstream gave was below 500, no when it was 500 or more or none came,
// unknown before any request.
Reachable string `json:"reachable"`
// Certificate is its outcome, issuer and expiry in words; ExpiresInDays is the whole days until not-after.
Certificate string `json:"certificate"`
ExpiresInDays *int `json:"expires_in_days,omitempty"`
LastAnswer *routestate.Answer `json:"last_answer,omitempty"`
}
func (f front) tools() []stdio.Tool {
node := map[string]any{"type": "string", "description": "only the proxy on this machine; every machine when left out"}
return []stdio.Tool{
{Name: RoutesTool,
Description: "Every site (web name, host name) the mesh's web front serves through route-proxy, the reverse proxy " +
"on the public machine: for each route the module and node it forwards to, its TLS certificate's issuer and " +
"expiry (days left), and the last answer upstream gave (status, time, error), so is X reachable and is its " +
"certificate good is answered without curl. Read from the proxy's routes state on the bus. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
Run: func(args map[string]any) (any, error) {
at, _ := args["node"].(string)
return f.list(at)
}},
{Name: RouteTool,
Description: "One site the mesh's web front (route-proxy) is asked about, in full: every rule it is served by " +
"(path, module, node, target, refusal, redirect, authentication), its certificate (issuer, not-after, last " +
"renewal and its outcome, or why it failed) and the last upstream answer. A name with no route says so, and " +
"when it was last asked for. Answers is X reachable, why does this site not load. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
"name": map[string]any{"type": "string", "description": "the site's name, such as app.example.org"},
"node": node}},
Run: func(args map[string]any) (any, error) {
name, _ := args["name"].(string)
at, _ := args["node"].(string)
return f.one(name, at)
}},
{Name: UnroutedTool,
Description: "Names the mesh's web front (route-proxy) answered \"no route for\": a site asked for in a request or " +
"a TLS handshake that no route serves, newest first, with when it was last asked and how often. Only names under a " +
"domain the proxy serves are kept by name, a few new ones a minute; every other refusal (a scan) is counted per " +
"proxy, the last minute's count and the total since its start. A site that should be served and is missing " +
"its route shows here. Kept a week. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": node}},
Run: func(args map[string]any) (any, error) {
at, _ := args["node"].(string)
return f.refusals(at)
}},
}
}
// read is every value of a state, of node's proxy alone when node is named, decoded into a fresh T each.
func read[T any](state reader, node string) ([]T, error) {
keys, err := state.Keys()
if err != nil {
return nil, err
}
var out []T
for _, key := range keys {
at, _ := routestate.Split(key)
if node != "" && at != node {
continue
}
entry, err := state.Get(key)
if err != nil {
return nil, err
}
if entry == nil {
continue // deleted between the list and the read
}
var v T
if err := json.Unmarshal(entry.Value, &v); err != nil {
return nil, fmt.Errorf("%s holds what is not a route-proxy record: %w", key, err)
}
out = append(out, v)
}
return out, nil
}
func (f front) list(node string) (map[string]any, error) {
routes, err := read[routestate.Route](f.routes, node)
if err != nil {
return nil, err
}
lines := make([]RouteLine, 0, len(routes))
unreachable, expiring := 0, 0
for _, r := range routes {
line := f.line(r)
if line.Reachable == "no" {
unreachable++
}
if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == routestate.Failed {
expiring++
}
lines = append(lines, line)
}
sort.Slice(lines, func(i, j int) bool {
if lines[i].Name != lines[j].Name {
return lines[i].Name < lines[j].Name
}
return lines[i].Proxy < lines[j].Proxy
})
return map[string]any{"routes": lines, "served": len(lines), "unreachable": unreachable,
"certificates_failing_or_within_14_days": expiring}, nil
}
func (f front) line(r routestate.Route) RouteLine {
line := RouteLine{Name: r.Name, Proxy: r.Proxy, Reach: r.Reach, LastAnswer: r.LastAnswer, Reachable: "unknown",
Targets: []string{}}
for _, t := range r.Targets {
line.Targets = append(line.Targets, describe(t))
}
if a := r.LastAnswer; a != nil {
line.Reachable = "yes"
if a.Error != "" || a.Status >= 500 || a.Status == 0 {
line.Reachable = "no"
}
}
c := r.Certificate
words := []string{}
if c.Outcome != "" {
words = append(words, c.Outcome)
}
if c.Issuer != "" {
words = append(words, "by "+c.Issuer)
}
if c.NotAfter != nil {
days := int(c.NotAfter.Sub(f.now()).Hours() / 24)
line.ExpiresInDays = &days
words = append(words, "until "+c.NotAfter.UTC().Format("2006-01-02"))
}
if c.Error != "" {
words = append(words, "("+c.Error+")")
}
if len(words) == 0 {
words = append(words, "none seen yet")
}
line.Certificate = c.Authority + ": " + strings.Join(words, " ")
return line
}
// describe is one rule in a few words: path → module@node at target, or what it answers with.
func describe(t routestate.Target) string {
path := t.Path
if path == "" {
path = "/"
}
var what string
switch {
case t.Deny:
what = "refused"
case t.Redirect != "":
what = "redirected to " + t.Redirect
default:
what = fmt.Sprintf("%s@%s at %s", t.Module, t.Node, t.Target)
}
if t.Sealed != "" {
what += " (authentication cannot be checked: " + t.Sealed + ")"
} else if t.Auth {
what += " (authenticated)"
}
return path + " → " + what
}
func (f front) one(name, node string) (map[string]any, error) {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return nil, fmt.Errorf("name is the site to read, such as app.example.org")
}
routes, err := read[routestate.Route](f.routes, node)
if err != nil {
return nil, err
}
var found []routestate.Route
for _, r := range routes {
if r.Name == name {
found = append(found, r)
}
}
if len(found) > 0 {
out := map[string]any{"name": name, "routed": true, "routes": found}
if len(found) == 1 {
out["reachable"] = f.line(found[0]).Reachable
}
return out, nil
}
asked, _, err := f.asked(node)
if err != nil {
return nil, err
}
for _, a := range asked {
if a.Name == name {
return map[string]any{"name": name, "routed": false, "asked": a,
"says": fmt.Sprintf("no proxy serves %s; the proxy on %s answered \"no route\" for it, last at %s",
name, a.Proxy, a.AskedAt.UTC().Format(time.RFC3339))}, nil
}
}
return map[string]any{"name": name, "routed": false,
"says": fmt.Sprintf("no proxy serves %s, and none was asked for it this week", name)}, nil
}
func (f front) refusals(node string) (map[string]any, error) {
asked, others, err := f.asked(node)
if err != nil {
return nil, err
}
sort.Slice(asked, func(i, j int) bool { return asked[i].AskedAt.After(asked[j].AskedAt) })
if asked == nil {
asked = []routestate.Unrouted{}
}
return map[string]any{"unrouted": asked, "names": len(asked), "other_refusals": others}, nil
}
// asked is the names refused and kept by name, and each proxy's count of the refusals not kept by name.
func (f front) asked(node string) ([]routestate.Unrouted, []routestate.Other, error) {
keys, err := f.unrouted.Keys()
if err != nil {
return nil, nil, err
}
names, others := keySet{}, keySet{}
for _, key := range keys {
if _, name := routestate.Split(key); name == routestate.OtherName {
others[key] = true
} else {
names[key] = true
}
}
asked, err := read[routestate.Unrouted](only{f.unrouted, names}, node)
if err != nil {
return nil, nil, err
}
counted, err := read[routestate.Other](only{f.unrouted, others}, node)
if counted == nil {
counted = []routestate.Other{}
}
return asked, counted, err
}
// keySet is a set of keys; only is a state narrowed to them.
type keySet map[string]bool
type only struct {
reader
keys keySet
}
func (o only) Keys() ([]string, error) {
out := make([]string, 0, len(o.keys))
for k := range o.keys {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
@@ -1,99 +0,0 @@
package main
import (
"encoding/json"
"sort"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// held is a state as the bus holds it, read-only.
type held map[string]any
func (h held) Keys() ([]string, error) {
out := []string{}
for k := range h {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (h held) Get(key string) (*stdio.StateEntry, error) {
v, ok := h[key]
if !ok {
return nil, nil
}
raw, err := json.Marshal(v)
return &stdio.StateEntry{Key: key, Value: raw}, err
}
// The web front's tools answer from the bus, in one call each (issue 774): every site with its target, certificate
// expiry and last answer; one site in full, or that it is not served and when it was asked for; and the names the
// proxy answered "no route for".
func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) {
now := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
expires := now.Add(30 * 24 * time.Hour)
routes := held{
"anchor.grafana.example": routestate.Route{Name: "grafana.example", Proxy: "anchor", Reach: routestate.ReachPublic,
Targets: []routestate.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}},
Certificate: routestate.Certificate{Authority: routestate.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: routestate.Issued},
LastAnswer: &routestate.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}},
"anchor.drive.example": routestate.Route{Name: "drive.example", Proxy: "anchor", Reach: routestate.ReachPublic,
Targets: []routestate.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}},
Certificate: routestate.Certificate{Authority: routestate.ReachPublic},
LastAnswer: &routestate.Answer{Status: 200, At: now}},
}
unrouted := held{
"anchor.old.example": routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3},
"anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1},
"anchor._other": routestate.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900},
}
tools := map[string]stdio.Tool{}
for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() {
tools[tool.Name] = tool
}
for _, row := range []struct {
tool string
args map[string]any
want []string // what the answer says, in its JSON
}{
{RoutesTool, nil, []string{`"served":2`, `"unreachable":1`,
`"name":"grafana.example"`, `"reachable":"no"`, `"expires_in_days":30`, `/ → grafana@home-server at http://10.0.0.2:3000`,
`"name":"drive.example"`, `"reachable":"yes"`, `public: none seen yet`}},
{RoutesTool, map[string]any{"node": "home-server"}, []string{`"served":0`}},
{RouteTool, map[string]any{"name": "Grafana.Example"}, []string{`"routed":true`, `"reachable":"no"`,
`connection refused`, `"issuer":"Let's Encrypt R11"`}},
{RouteTool, map[string]any{"name": "wiki.example"}, []string{`"routed":false`,
`answered \"no route\" for it, last at 2026-10-30T21:00:00Z`}},
{RouteTool, map[string]any{"name": "nothing.example"}, []string{`"routed":false`, `none was asked for it`}},
{UnroutedTool, nil, []string{`"names":2`, `"unrouted":[{"name":"wiki.example"`,
`"other_refusals":[{"proxy":"anchor"`, `"last_minute":40`, `"since_start":900`}},
} {
t.Run(row.tool, func(t *testing.T) {
tool, ok := tools[row.tool]
if !ok {
t.Fatalf("%s is not served", row.tool)
}
if !strings.Contains(tool.Description, "route-proxy") {
t.Errorf("%s's description does not name the web front, so a search does not find it", row.tool)
}
answer, err := tool.Run(row.args)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(answer)
for _, w := range row.want {
if !strings.Contains(string(raw), w) {
t.Errorf("%s(%v) does not say %s:\n%s", row.tool, row.args, w, raw)
}
}
})
}
}
@@ -19,57 +19,55 @@ const (
// path back: nothing is re-registered, no order reaches the CA, and issuance stops with nothing
// saying why. Naming the cache after the authority means the account is only ever found where it is
// still valid — the new root lands in a directory with no account in it, and autocert registers.
func TestTheAccountCacheIsNamedAfterItsAuthority(t *testing.T) {
t.Run("a new CA root means a new account cache", func(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot))
if before == after {
t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before)
}
})
func TestANewCARootMeansANewAccountCache(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
before := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
after := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(secondRoot))
if before == after {
t.Fatalf("a re-initialised CA reuses the account it was rejected for: %s", before)
}
}
// And the SAME authority keeps the account it registered, restart after restart.
//
// This is the whole reason ACME_CACHE is required in the first place: an account and its
// certificates that did not persist would be re-ordered on every restart, which works silently until
// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the
// mesh writes that file, and a trailing newline coming or going must not throw away an account.
t.Run("the same authority keeps its account", func(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n"))
if first != again {
t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s",
first, again)
}
})
// And the SAME authority keeps the account it registered, restart after restart.
//
// This is the whole reason ACME_CACHE is required in the first place: an account and its
// certificates that did not persist would be re-ordered on every restart, which works silently until
// a rate limit says it does not. Whitespace around the delivered root is not a new authority — the
// mesh writes that file, and a trailing newline coming or going must not throw away an account.
func TestTheSameAuthorityKeepsItsAccount(t *testing.T) {
const directory = "https://anchor.internal/acme/acme/directory"
first := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte(firstRoot))
again := forThisAuthority("/var/lib/route-proxy/acme", directory, []byte("\n"+firstRoot+"\n\n"))
if first != again {
t.Errorf("the same authority was given two caches, so every restart orders again:\n%s\n%s",
first, again)
}
}
// Staging and production are different authorities, and were sharing one account.
//
// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against
// staging reused the staging account, because the cache had no idea they were different.
t.Run("staging and production do not share an account", func(t *testing.T) {
staging := forThisAuthority("/acme", stagingDirectory, nil)
production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil)
if staging == production {
t.Errorf("two issuers share one account: %s", staging)
}
})
// Staging and production are different authorities, and were sharing one account.
//
// The latent fault of the same shape: pointing ACME_DIRECTORY at production after testing against
// staging reused the staging account, because the cache had no idea they were different.
func TestStagingAndProductionDoNotShareAnAccount(t *testing.T) {
staging := forThisAuthority("/acme", stagingDirectory, nil)
production := forThisAuthority("/acme", "https://acme-v02.api.letsencrypt.org/directory", nil)
if staging == production {
t.Errorf("two issuers share one account: %s", staging)
}
}
// It stays inside the directory the mesh gave it, and is a plain name.
//
// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or
// that carried a separator out of the PEM — would put an account somewhere nothing persists.
t.Run("the account cache stays where the mesh put it", func(t *testing.T) {
const cache = "/var/lib/route-proxy/acme"
got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot))
if !strings.HasPrefix(got, cache+"/") {
t.Fatalf("the account cache is not under %s: %s", cache, got)
}
name := strings.TrimPrefix(got, cache+"/")
if name != filepath.Base(got) || strings.ContainsAny(name, "/.") {
t.Errorf("the account cache is not a plain name: %q", name)
}
})
// It stays inside the directory the mesh gave it, and is a plain name.
//
// The mesh owns ACME_CACHE and mounts it; a name derived from a certificate that escaped it — or
// that carried a separator out of the PEM — would put an account somewhere nothing persists.
func TestTheAccountCacheStaysWhereTheMeshPutIt(t *testing.T) {
const cache = "/var/lib/route-proxy/acme"
got := forThisAuthority(cache, "https://anchor.internal/acme/acme/directory", []byte(firstRoot))
if !strings.HasPrefix(got, cache+"/") {
t.Fatalf("the account cache is not under %s: %s", cache, got)
}
name := strings.TrimPrefix(got, cache+"/")
if name != filepath.Base(got) || strings.ContainsAny(name, "/.") {
t.Errorf("the account cache is not a plain name: %q", name)
}
}
+40 -83
View File
@@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"log"
"strings"
@@ -9,47 +10,32 @@ import (
"github.com/nats-io/nats.go"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provisions"
"route-proxy/internal/routestate"
)
// keptStates is every state route-proxy keeps, as both its halves write them: the routes and the refusals this
// proxy writes, readable by others, and the bound entries the provider helper writes.
func keptStates() []announce.State {
return []announce.State{
{Name: routestate.BoundState},
{Name: routestate.RoutesState, Readable: true},
{Name: routestate.UnroutedState, Readable: true},
}
}
// What the mesh says this proxy routes, and who is inside the mesh, read on the bus.
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
//
// **The routes are the seat mesh-provisioner's plans, alone** (issue 774, research 072 row 34): every plan's entry
// naming this proxy as a route's provider, followed, so a route added or withdrawn reaches a running proxy without a
// restart. **Who is inside** is this proxy's membership (novox/hq ADR 0160, ADR 0167): every machine's address on the
// private network, which is who may be served an internal name; until it is read, an internal name is served to this
// machine alone.
// **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
// the same contributions its file is written from — and every machine's address on the private
// network, which is who may be served an internal name. Read once at connect and followed, so a
// route added or a machine joining reaches a running proxy without a restart.
// follow connects with the credential in path, follows the plans and the membership, and retries the connection for as
// long as it takes: until the bus answers, the proxy routes nothing, and says so.
func follow(path string, held *table) {
// followMembership connects with the credential in path and applies every membership the mesh
// issues this proxy. It retries the first connection for as long as it takes: a proxy that started
// before the bus keeps serving the file, and takes the bus when it answers.
func followMembership(path string, held *table, fromBus *atomic.Bool) {
for {
err := followOnce(path, held)
err := followOnce(path, held, fromBus)
if err == nil {
return
}
log.Printf("cannot follow the plans that say what this proxy routes, so it routes what it last read: %v", err)
log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err)
time.Sleep(30 * time.Second)
}
}
func followOnce(path string, held *table) error {
func followOnce(path string, held *table, fromBus *atomic.Bool) error {
account, err := process.ReadAccount(path)
if err != nil {
return err
@@ -58,71 +44,42 @@ func followOnce(path string, held *table) error {
if err != nil {
return fmt.Errorf("connecting to the bus at %s: %w", membership.BareAddress(account.URL), err)
}
// What this proxy keeps, said on the bus (novox/hq ADR 0495; issue 774): the names it serves and the names it
// refused, readable by any module granted a read of them; the provider helper's entries, read by the machines'
// clients through their own grant. Said before the first write, so no write of an unannounced state is refused.
if err := conn.Keeps(keptStates()...); err != nil {
log.Printf("cannot say on the bus what this proxy keeps, so it keeps nothing there: %v", err)
} else {
held.ledger.attach(account.Node, conn.State(routestate.RoutesState), conn.State(routestate.UnroutedState))
}
if err := followPlans(conn.Read(provisions.PlansRead), newPlanned(account.Node), held); err != nil {
subject := membership.Subject(account.Node, account.Module)
err = conn.FollowMembership(func(issued membership.Membership) {
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
log.Printf("routes now come from this proxy's membership on %s", subject)
}
}, func(err error) { log.Print(err) })
if err != nil {
conn.Close()
return err
}
subject := membership.Subject(account.Node, account.Module)
err = conn.FollowMembership(func(issued membership.Membership) {
applyMembership(issued, held)
}, func(err error) { log.Print(err) })
if err != nil {
log.Printf("cannot follow this proxy's membership on %s, so an internal name is served to this machine "+
"alone: %v", subject, err)
}
return nil
}
// planWatch is the seat's plans as this proxy watches them: every current key, then every change.
type planWatch interface {
Watch(key string, handle func(stdio.StateChange) error) error
}
// followPlans serves what the plans give this proxy: the whole table once every current plan is read, then again on
// every change that alters it. A table built from part of the plans would withdraw every route not yet read.
func followPlans(plans planWatch, given *planned, held *table) error {
var read atomic.Bool // every current plan is read: from here, each change is served as it comes
err := plans.Watch("", func(c stdio.StateChange) error {
var value []byte
if c.Op != "delete" {
value = c.Value
}
if changed := given.take(c.Key, value); changed && read.Load() {
serve(given, held)
}
return nil
})
if err != nil {
return fmt.Errorf("watching %s: %w", provisions.PlansRead, err)
}
read.Store(true)
serve(given, held)
return nil
}
// serve replaces what the proxy serves with what the plans give it.
func serve(given *planned, held *table) {
routes, public := routesOf(given.contributions())
held.set(routes, public)
log.Printf("serving %d route(s) from the seat's plans: %s", len(routes), strings.Join(held.names(), ", "))
}
// applyMembership takes who is inside the mesh from a membership, and says whether it could read it.
// applyMembership serves what a membership says, and says whether it said anything about routes.
//
// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays
// the source rather than every route being withdrawn because a field was absent.
func applyMembership(issued membership.Membership, held *table) bool {
inside, err := sourcesOf(issued.Mesh)
if err != nil {
log.Printf("the mesh in this proxy's membership is unreadable, keeping who is inside as it was: %v", err)
raw, carries := issued.Receives["route"]
if !carries {
return false
}
var contributions []contribution
if err := json.Unmarshal(raw, &contributions); err != nil {
log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err)
return false
}
inside, err := sourcesOf(issued.Mesh)
if err != nil {
log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err)
return false
}
routes, public := routesOf(contributions)
held.set(routes, public)
held.setInside(inside)
log.Printf("internal names are served to %d machine(s)", len(inside))
log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s",
len(routes), len(inside), strings.Join(held.names(), ", "))
return true
}
@@ -1,552 +0,0 @@
package main
import (
"context"
"crypto/x509"
"encoding/json"
"encoding/pem"
"log"
"net/http"
"reflect"
"sort"
"strings"
"sync"
"time"
"golang.org/x/crypto/acme/autocert"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// What this proxy says on the bus about the names it serves and the names it refuses (issue 774).
//
// **On change, never on a timer.** A route arriving or leaving, a certificate issued or failing, an upstream answer
// that differs from the last one written, a name asked for that is not served: each marks its key, and one writer
// puts the marked keys. An answer that is the same as the last one written is written again only once it is older
// than refreshAfter, so a site answering 200 all day costs one write a minute, not one per request.
//
// Until the bus is reached nothing is written; what happened meanwhile is held and written when it is.
// refreshAfter is how old a written answer, refusal or certificate failure may be before the same one is written
// again with its new time.
const refreshAfter = time.Minute
// mostAsked bounds the names held in memory as asked and not served: a scan sends any name it likes.
const mostAsked = 4096
// newPerMinute is how many names not seen before are kept by name in a minute; past it they are only counted.
const newPerMinute = 5
// kvStore is one of the module's states, as process.State is one.
type kvStore interface {
Put(key string, value any) (uint64, error)
Delete(key string) error
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// ledger is what this proxy keeps of what it serves and refuses, and writes to the bus.
type ledger struct {
mu sync.Mutex
now func() time.Time
node string
routes, unrouted kvStore
served map[string]*routestate.Route
previous map[string]routestate.Route // what the bus held for this machine when it was reached
asked map[string]*routestate.Unrouted
answerWritten map[string]time.Time
askedWritten map[string]time.Time
dirtyRoutes map[string]bool // a name to put when served, to delete when not
dirtyAsked map[string]bool
// domains are the registrable domains of the names served: a refused name under one is kept by name.
domains map[string]bool
newMinute time.Time
newInMinute int
other routestate.Other
otherMinute time.Time
otherInMinute int
otherDirty bool
routesFailing, askFailed bool // a failed write said once, until one succeeds
kick chan struct{}
}
func newLedger() *ledger {
return &ledger{
now: time.Now,
served: map[string]*routestate.Route{},
previous: map[string]routestate.Route{},
asked: map[string]*routestate.Unrouted{},
answerWritten: map[string]time.Time{},
askedWritten: map[string]time.Time{},
dirtyRoutes: map[string]bool{},
dirtyAsked: map[string]bool{},
domains: map[string]bool{},
kick: make(chan struct{}, 1),
}
}
// loop writes whatever was marked, each time something was.
func (l *ledger) loop() {
for range l.kick {
l.flush()
}
}
func (l *ledger) wake() {
select {
case l.kick <- struct{}{}:
default:
}
}
// attach is the bus reached, as node: what the bus held for this machine is read back, so a restart keeps a name's
// certificate and last answer, and everything held is written.
func (l *ledger) attach(node string, routes, unrouted kvStore) {
if l == nil {
return
}
before := map[string]routestate.Route{}
if keys, err := routes.Keys(); err != nil {
log.Printf("cannot read back what this proxy said it serves: %v", err)
} else {
for _, key := range keys {
at, name := routestate.Split(key)
if at != node {
continue
}
entry, err := routes.Get(key)
if err != nil || entry == nil {
continue
}
var r routestate.Route
if json.Unmarshal(entry.Value, &r) == nil {
before[name] = r
}
}
}
l.mu.Lock()
l.node, l.routes, l.unrouted = node, routes, unrouted
for name, r := range before {
l.previous[name] = r
l.dirtyRoutes[name] = true // put again when served, deleted when no longer
if now := l.served[name]; now != nil {
keepWhatWasKnown(now, r)
}
}
for name := range l.served {
l.dirtyRoutes[name] = true
}
for name := range l.asked {
l.dirtyAsked[name] = true
}
l.mu.Unlock()
l.wake()
}
// keepWhatWasKnown carries a certificate and a last answer read back from the bus onto a route that knows none yet.
func keepWhatWasKnown(r *routestate.Route, before routestate.Route) {
if r.Certificate.Outcome == "" && r.Certificate.NotAfter == nil && before.Certificate.Authority == r.Certificate.Authority {
r.Certificate = before.Certificate
}
if r.LastAnswer == nil {
r.LastAnswer = before.LastAnswer
}
}
// serving is the table this proxy now serves: a name new or changed is written, a name gone is deleted, and a name
// that was asked for unserved and now is served leaves the unrouted state.
func (l *ledger) serving(routes map[string][]rule, public map[string]bool) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
for name, rules := range routes {
reach := routestate.ReachInternal
if public[name] {
reach = routestate.ReachPublic
}
targets := targetsOf(rules)
r := l.served[name]
if r == nil {
r = &routestate.Route{Name: name}
if before, ok := l.previous[name]; ok {
r.Certificate.Authority = reach
keepWhatWasKnown(r, before)
}
l.served[name] = r
l.dirtyRoutes[name] = true
}
if r.Reach != reach || !reflect.DeepEqual(r.Targets, targets) {
r.Reach, r.Targets = reach, targets
l.dirtyRoutes[name] = true
}
if r.Certificate.Authority != reach {
r.Certificate = routestate.Certificate{Authority: reach}
l.dirtyRoutes[name] = true
}
if _, was := l.asked[name]; was {
delete(l.asked, name)
l.dirtyAsked[name] = true
}
}
l.domains = map[string]bool{}
for name := range routes {
if d := routestate.Registrable(name); d != "" {
l.domains[d] = true
}
}
for name := range l.served {
if _, still := routes[name]; !still {
delete(l.served, name)
delete(l.answerWritten, name)
l.dirtyRoutes[name] = true
}
}
l.wake()
}
func targetsOf(rules []rule) []routestate.Target {
out := make([]routestate.Target, 0, len(rules))
for _, r := range rules {
out = append(out, routestate.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target,
Deny: r.policy.deny, Redirect: r.policy.redirectTo, Auth: len(r.policy.users) > 0 || r.policy.sealed != "",
Sealed: r.policy.sealed})
}
return out
}
// answered is an upstream answer to a request for name: its status, or the error that stood in for one.
func (l *ledger) answered(name, path string, status int, failed error) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
why := ""
if failed != nil {
why = failed.Error()
}
before := r.LastAnswer
r.LastAnswer = &routestate.Answer{Status: status, At: now, Error: why, Path: path}
if before != nil && before.Status == status && before.Error == why && now.Sub(l.answerWritten[name]) < refreshAfter {
return
}
l.answerWritten[name] = now
l.dirtyRoutes[name] = true
l.wake()
}
// refused is a name this proxy answered "no route for": in a request, or in a handshake's server name.
//
// **Only a name under a domain this proxy serves is kept by name**, and at most newPerMinute new ones a minute: a scan
// sends any name it likes, and each kept name is a key on the bus. Every other refusal is counted, and the count is
// written once a minute under the proxy's OtherName key. Nothing here scans: forgetting old names is the writer's.
// Who asked is not kept: the journal line beside it is what the machine's intrusion prevention reads.
func (l *ledger) refused(name string, internalOnly, handshake bool) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
if _, isServed := l.served[name]; isServed && !internalOnly {
return
}
now := l.now()
a := l.asked[name]
if a == nil {
if !routestate.Keyable(name) || !l.domains[routestate.Registrable(name)] || !l.mayAddName(now) {
l.countOther(now)
return
}
a = &routestate.Unrouted{Name: name}
l.asked[name] = a
}
a.AskedAt, a.InternalOnly, a.Handshake = now, internalOnly, handshake
a.Times++
if a.Times > 1 && now.Sub(l.askedWritten[name]) < refreshAfter {
return
}
l.askedWritten[name] = now
l.dirtyAsked[name] = true
l.wake()
}
// mayAddName says whether another new name may be kept this minute, and counts it.
func (l *ledger) mayAddName(now time.Time) bool {
if minute := now.Truncate(time.Minute); !minute.Equal(l.newMinute) {
l.newMinute, l.newInMinute = minute, 0
}
if l.newInMinute >= newPerMinute {
return false
}
l.newInMinute++
return true
}
// countOther counts a refusal not kept by name. A minute's count is written when the next minute's first one comes.
func (l *ledger) countOther(now time.Time) {
minute := now.Truncate(time.Minute)
if !minute.Equal(l.otherMinute) {
if l.otherInMinute > 0 {
l.other.Minute, l.other.LastMinute = l.otherMinute, l.otherInMinute
l.otherDirty = true
l.wake()
}
l.otherMinute, l.otherInMinute = minute, 0
}
l.otherInMinute++
l.other.SinceStart++
}
// forgetOldestAsked drops the names asked longest ago from memory until mostAsked are left; the bus keeps them until
// their time to live ends. Called by the writer, never on a request.
func (l *ledger) forgetOldestAsked() {
if len(l.asked) <= mostAsked {
return
}
names := make([]string, 0, len(l.asked))
for name := range l.asked {
names = append(names, name)
}
sort.Slice(names, func(i, j int) bool { return l.asked[names[i]].AskedAt.Before(l.asked[names[j]].AskedAt) })
for _, name := range names[:len(names)-mostAsked] {
delete(l.asked, name)
delete(l.askedWritten, name)
delete(l.dirtyAsked, name)
}
}
// certified is a certificate an authority issued for name, as autocert stored it.
func (l *ledger) certified(name string, leaf *x509.Certificate) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
notAfter := leaf.NotAfter.UTC()
r.Certificate.Issuer, r.Certificate.NotAfter = issuerOf(leaf), &notAfter
r.Certificate.RenewedAt, r.Certificate.At = &now, &now
r.Certificate.Outcome, r.Certificate.Error = routestate.Issued, ""
l.dirtyRoutes[name] = true
l.wake()
}
// handedOut is the certificate a handshake for name was answered with: written when it is not the one recorded.
func (l *ledger) handedOut(name string, leaf *x509.Certificate) {
if l == nil || leaf == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
notAfter := leaf.NotAfter.UTC()
if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != routestate.Failed {
return
}
now := l.now()
r.Certificate.Issuer, r.Certificate.NotAfter, r.Certificate.At = issuerOf(leaf), &notAfter, &now
if r.Certificate.Outcome != routestate.Issued || r.Certificate.Error != "" {
r.Certificate.Outcome, r.Certificate.Error = routestate.Cached, ""
}
l.dirtyRoutes[name] = true
l.wake()
}
// certificateFailed is a handshake for name that found no certificate to answer with, and why.
func (l *ledger) certificateFailed(name string, failed error) {
if l == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
r := l.served[name]
if r == nil {
return
}
now := l.now()
same := r.Certificate.Outcome == routestate.Failed && r.Certificate.Error == failed.Error()
if same && r.Certificate.At != nil && now.Sub(*r.Certificate.At) < refreshAfter {
return
}
r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = routestate.Failed, failed.Error(), &now
l.dirtyRoutes[name] = true
l.wake()
}
func issuerOf(leaf *x509.Certificate) string {
if cn := leaf.Issuer.CommonName; cn != "" {
if len(leaf.Issuer.Organization) > 0 {
return leaf.Issuer.Organization[0] + " " + cn
}
return cn
}
return leaf.Issuer.String()
}
// flush writes every marked key. A route's write that fails stays marked and is tried again at the next change; a
// refused name's write that fails is dropped, never tried again: the bucket refusing it is full, or the bus is away,
// and either way it is not worth a storm of retries. Each kind of failure is logged once, until a write succeeds.
func (l *ledger) flush() {
type put struct {
store kvStore
key string
value any // nil deletes
route bool
}
l.mu.Lock()
if l.routes == nil {
l.mu.Unlock()
return
}
l.forgetOldestAsked()
var puts []put
for name := range l.dirtyRoutes {
p := put{store: l.routes, key: routestate.Key(l.node, name), route: true}
if r := l.served[name]; r != nil {
c := *r
c.Proxy = l.node
p.value = c
}
puts = append(puts, p)
}
for name := range l.dirtyAsked {
p := put{store: l.unrouted, key: routestate.Key(l.node, name)}
if a := l.asked[name]; a != nil {
c := *a
c.Proxy = l.node
p.value = c
}
puts = append(puts, p)
}
if l.otherDirty {
o := l.other
o.Proxy = l.node
puts = append(puts, put{store: l.unrouted, key: routestate.Key(l.node, routestate.OtherName), value: o})
}
l.dirtyRoutes, l.dirtyAsked, l.otherDirty = map[string]bool{}, map[string]bool{}, false
l.mu.Unlock()
for _, p := range puts {
var err error
if p.value == nil {
err = p.store.Delete(p.key)
} else {
_, err = p.store.Put(p.key, p.value)
}
l.mu.Lock()
switch {
case err == nil && p.route:
l.routesFailing = false
case err == nil:
l.askFailed = false
case p.route:
if !l.routesFailing {
log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err)
}
l.routesFailing = true
_, name := routestate.Split(p.key)
l.dirtyRoutes[name] = true
default:
if !l.askFailed {
log.Printf("cannot write %s to the bus, dropped with every refused name until a write succeeds: %v",
p.key, err)
}
l.askFailed = true
}
l.mu.Unlock()
}
}
// recordingCache is an authority's certificate cache that tells the ledger of every certificate stored in it: the
// one place autocert says it was issued one.
type recordingCache struct {
autocert.Cache
held *table
}
func (c recordingCache) Put(ctx context.Context, key string, data []byte) error {
if err := c.Cache.Put(ctx, key, data); err != nil {
return err
}
name := strings.TrimSuffix(key, "+rsa")
if strings.Contains(name, "+") || !c.held.routed(name) {
return nil // the account key, a challenge token
}
if leaf := leafOf(data); leaf != nil {
c.held.ledger.certified(bareHost(name), leaf)
}
return nil
}
// leafOf is the first certificate in what autocert stores: the private key, then the chain, in PEM.
func leafOf(data []byte) *x509.Certificate {
for {
var block *pem.Block
block, data = pem.Decode(data)
if block == nil {
return nil
}
if block.Type == "CERTIFICATE" {
leaf, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil
}
return leaf
}
}
}
// answerRecorder carries a proxied request's answer through and remembers its status and the proxy's error.
type answerRecorder struct {
http.ResponseWriter
status int
err error
}
func (a *answerRecorder) WriteHeader(status int) {
if a.status == 0 {
a.status = status
}
a.ResponseWriter.WriteHeader(status)
}
func (a *answerRecorder) Write(b []byte) (int, error) {
if a.status == 0 {
a.status = http.StatusOK
}
return a.ResponseWriter.Write(b)
}
// Unwrap lets http.ResponseController reach the connection beneath: a flush, an upgrade's hijack.
func (a *answerRecorder) Unwrap() http.ResponseWriter { return a.ResponseWriter }
// proxyFailed is the reverse proxy's answer when upstream gave none: the error kept for the ledger, and a 502.
func proxyFailed(w http.ResponseWriter, r *http.Request, err error) {
if a, ok := w.(*answerRecorder); ok {
a.err = err
}
log.Printf("http: proxy error for %s%s: %v", r.Host, r.URL.Path, err)
w.WriteHeader(http.StatusBadGateway)
}
@@ -1,291 +0,0 @@
package main
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"net/http"
"net/http/httptest"
"sort"
"strings"
"testing"
"time"
"golang.org/x/crypto/acme/autocert"
stdio "git.novox.be/novox/mesh-sdk/go"
"route-proxy/internal/routestate"
)
// fakeState is a state on the bus, in memory, counting what was written to it.
type fakeState struct {
values map[string]json.RawMessage
puts int
tries int // every Put asked, refused or not
fail error // what every Put is refused with, while set
}
func newFakeState() *fakeState { return &fakeState{values: map[string]json.RawMessage{}} }
func (f *fakeState) Put(key string, value any) (uint64, error) {
f.tries++
if f.fail != nil {
return 0, f.fail
}
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
f.values[key] = raw
f.puts++
return uint64(f.puts), nil
}
func (f *fakeState) Delete(key string) error { delete(f.values, key); return nil }
func (f *fakeState) Keys() ([]string, error) {
out := []string{}
for k := range f.values {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (f *fakeState) Get(key string) (*stdio.StateEntry, error) {
v, ok := f.values[key]
if !ok {
return nil, nil
}
return &stdio.StateEntry{Key: key, Value: v}, nil
}
func (f *fakeState) route(t *testing.T, key string) *routestate.Route {
t.Helper()
raw, ok := f.values[key]
if !ok {
return nil
}
var r routestate.Route
if err := json.Unmarshal(raw, &r); err != nil {
t.Fatal(err)
}
return &r
}
// issuedPEM is what autocert stores for a name: the key, then the certificate, signed by an issuer named here.
func issuedPEM(t *testing.T, name string, notAfter time.Time) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{SerialNumber: big.NewInt(1), DNSNames: []string{name},
Subject: pkix.Name{CommonName: name}, Issuer: pkix.Name{CommonName: "R11", Organization: []string{"Let's Encrypt"}},
NotBefore: notAfter.Add(-90 * 24 * time.Hour), NotAfter: notAfter}
issuer := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: template.Issuer, NotBefore: template.NotBefore,
NotAfter: notAfter, IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
keyDER, _ := x509.MarshalECPrivateKey(key)
out := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
return append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
}
// What the proxy tells the bus (issue 774): each served name with where it goes, its certificate and the last
// upstream answer; each name refused; and only on change. Without the ledger nothing reaches the bus, and the
// operator's question "is grafana reachable" is answered with curl.
func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) {
notAfter := time.Date(2027, 1, 28, 12, 0, 0, 0, time.UTC)
gone := func() string { // an address nothing listens on: the upstream that does not answer
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
return l.Addr().String()
}()
_, port, _ := net.SplitHostPort(gone)
contributions := []contribution{
{From: "grafana", Node: "anchor", At: "127.0.0.1", Values: map[string]any{"name": "grafana.example", "port": mustFloat(t, port)}},
{From: "drive", Node: "home-server", At: "10.0.0.2", Values: map[string]any{"internal-name": "drive.internal", "port": 8080.0}},
}
for _, row := range []struct {
name string
act func(t *testing.T, held *table)
check func(t *testing.T, routes, unrouted *fakeState)
}{
{"a served name is written with the module and node it forwards to",
func(t *testing.T, held *table) {},
func(t *testing.T, routes, _ *fakeState) {
r := routes.route(t, "anchor.grafana.example")
if r == nil || r.Proxy != "anchor" || r.Reach != routestate.ReachPublic || len(r.Targets) != 1 ||
r.Targets[0].Module != "grafana" || r.Targets[0].Node != "anchor" || r.Targets[0].Target != "http://127.0.0.1:"+port {
t.Fatalf("grafana's route is not on the bus as served: %+v", r)
}
if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != routestate.ReachInternal ||
d.Certificate.Authority != routestate.ReachInternal {
t.Fatalf("an internal-only name is not on the bus as internal: %+v", d)
}
}},
{"a withdrawn route is deleted",
func(t *testing.T, held *table) {
routes, public := routesOf(contributions[1:])
held.set(routes, public)
},
func(t *testing.T, routes, _ *fakeState) {
if routes.route(t, "anchor.grafana.example") != nil {
t.Fatal("a route no longer served is still on the bus")
}
}},
{"an upstream that does not answer is written as a 502 with why, once a minute",
func(t *testing.T, held *table) {
for range 3 {
req := httptest.NewRequest(http.MethodGet, "http://grafana.example/login", nil)
handler(held).ServeHTTP(httptest.NewRecorder(), req)
}
},
func(t *testing.T, routes, _ *fakeState) {
r := routes.route(t, "anchor.grafana.example")
if r == nil || r.LastAnswer == nil || r.LastAnswer.Status != http.StatusBadGateway ||
r.LastAnswer.Error == "" || r.LastAnswer.Path != "/login" {
t.Fatalf("the failed answer is not on the bus: %+v", r)
}
if routes.puts != 3 { // the two routes, then the first answer; the same answer again is not written
t.Fatalf("the same answer was written %d times in all, want 3 writes", routes.puts)
}
}},
{"a name under a served domain is written as asked, without who asked, and leaves when it is served",
func(t *testing.T, held *table) {
req := httptest.NewRequest(http.MethodGet, "http://wiki.grafana.example/", nil)
req.RemoteAddr = "192.0.2.7:51000"
handler(held).ServeHTTP(httptest.NewRecorder(), req)
held.ledger.flush()
var a routestate.Unrouted
raw := held.ledger.unrouted.(*fakeState).values["anchor.wiki.grafana.example"]
if err := json.Unmarshal(raw, &a); err != nil || a.Times != 1 || a.AskedAt.IsZero() {
t.Fatalf("the refused name is not on the bus as asked: %s", raw)
}
if strings.Contains(string(raw), "192.0.2.7") || strings.Contains(string(raw), `"from"`) {
t.Fatalf("who asked is on the bus: %s", raw)
}
routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor",
Values: map[string]any{"name": "wiki.grafana.example", "port": 3000.0}}))
held.set(routes, public)
},
func(t *testing.T, _, unrouted *fakeState) {
if len(unrouted.values) != 0 {
t.Fatalf("a name now served is still written as unrouted: %v", unrouted.values)
}
}},
{"a handshake for a name not routed is written as asked",
func(t *testing.T, held *table) {
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return nil, errors.New("no public route for \"forgotten.grafana.example\" in this mesh, so no certificate is asked for")
}, nil)
_, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.grafana.example"})
},
func(t *testing.T, _, unrouted *fakeState) {
var a routestate.Unrouted
if err := json.Unmarshal(unrouted.values["anchor.forgotten.grafana.example"], &a); err != nil || !a.Handshake {
t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values)
}
}},
{"a flood of names writes at most the limit by name, counts the rest once a minute, and never retries a refused write",
func(t *testing.T, held *table) {
clock := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC)
held.ledger.now = func() time.Time { return clock }
full := held.ledger.unrouted.(*fakeState)
for i := range 40 {
held.ledger.refused(fmt.Sprintf("x%d.grafana.example", i), false, false) // under a served domain
held.ledger.refused(fmt.Sprintf("x%d.scan.test", i), false, true) // under none
}
held.ledger.flush()
if named := len(full.values); named != newPerMinute {
t.Fatalf("a flood wrote %d names, want the limit %d: %v", named, newPerMinute, full.values)
}
full.fail = errors.New("nats: maximum bytes exceeded")
clock = clock.Add(time.Minute)
held.ledger.refused("y.grafana.example", false, false) // a new minute: one more name,
held.ledger.refused("v.scan.test", false, false) // and the last minute's count
held.ledger.flush()
tries := full.tries
held.ledger.refused("y.grafana.example", false, false)
held.ledger.refused("z.scan.test", false, false)
held.ledger.flush()
if full.tries != tries {
t.Fatalf("writes the bucket refused were tried again: %d tries, then %d", tries, full.tries)
}
full.fail = nil
clock = clock.Add(time.Minute)
held.ledger.refused("w.scan.test", false, false)
},
func(t *testing.T, _, unrouted *fakeState) {
var o routestate.Other
if err := json.Unmarshal(unrouted.values["anchor._other"], &o); err != nil || o.LastMinute != 2 || o.SinceStart != 78 {
t.Fatalf("the refusals not kept by name are not counted on the bus: %s", unrouted.values["anchor._other"])
}
}},
{"a certificate an authority issued is written with its issuer, expiry and renewal",
func(t *testing.T, held *table) {
cache := recordingCache{Cache: autocert.DirCache(t.TempDir()), held: held}
if err := cache.Put(context.Background(), "grafana.example", issuedPEM(t, "grafana.example", notAfter)); err != nil {
t.Fatal(err)
}
_ = cache.Put(context.Background(), "acme_account+key", []byte("not a certificate"))
},
func(t *testing.T, routes, _ *fakeState) {
c := routes.route(t, "anchor.grafana.example").Certificate
if c.Outcome != routestate.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil ||
!c.NotAfter.Equal(notAfter) || c.RenewedAt == nil {
t.Fatalf("the issued certificate is not on the bus: %+v", c)
}
}},
{"a certificate that cannot be had is written with why",
func(t *testing.T, held *table) {
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return nil, errors.New("acme: urn:ietf:params:acme:error:rateLimited")
}, nil)
_, _ = pick(&tls.ClientHelloInfo{ServerName: "grafana.example"})
},
func(t *testing.T, routes, _ *fakeState) {
c := routes.route(t, "anchor.grafana.example").Certificate
if c.Outcome != routestate.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" {
t.Fatalf("the failed certificate is not on the bus: %+v", c)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
held := newTable()
held.ledger = newLedger()
routes, unrouted := newFakeState(), newFakeState()
held.ledger.attach("anchor", routes, unrouted)
table, public := routesOf(contributions)
held.set(table, public)
held.ledger.flush()
row.act(t, held)
held.ledger.flush()
row.check(t, routes, unrouted)
})
}
}
func mustFloat(t *testing.T, port string) float64 {
t.Helper()
var n float64
if err := json.Unmarshal([]byte(port), &n); err != nil {
t.Fatal(err)
}
return n
}
+89 -78
View File
@@ -1,14 +1,13 @@
// A reverse proxy, in the form the mesh expects one.
//
// **A route is a provision** (novox/hq ADR 0007, 08-connectivity §3; issue 774, research 072 row 34). A
// module that must be reachable declares it requires `route` and contributes the name it wants; the
// proxy provides `route` and serves every consumer whose plan, in the seat mesh-provisioner's state
// `plans`, names it as the route's provider. It is the mirror of a database grant: there the consumer
// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable
// declares it requires `route` and contributes the name it wants; the proxy provides `route` and
// receives every consumer that asked. It is the mirror of a database grant: there the consumer
// supplies a name and receives credentials, here it supplies a target and receives a name.
//
// **It is an example, not part of the controller.** The controller decides and never touches a
// machine. A real deployment runs whatever proxy it likes — the contract is the plan's entry, not
// this program. What lives here is that contract, written as something that runs so it can be read
// **It is an example, not part of the control plane.** The control plane decides and never touches
// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this
// program. What lives here is that contract, written as something that runs so it can be read
// rather than described.
//
// **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used
@@ -17,9 +16,11 @@
// an open middleware surface recreates the thing being replaced and is far harder to narrow later
// than a closed one is to widen.
//
// What it is given, in each plan's entry for the route: every contribution the consumer makes,
// composed by the controller, and where the consumer's machine is on the private network (plans.go).
// Each contribution's values carry the name and port, and optionally:
// What it is given, written by the host from an ordinary declaration:
//
// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is
//
// Each contribution's values carry the name and port as before, and optionally:
//
// path the path prefix this rule is scoped to; absent means every path
// priority which rule wins where two match; higher first, and the order is total
@@ -34,8 +35,8 @@
// outright rather than served unprotected, and a secret that cannot be read makes the route refuse
// rather than open — a gate that cannot check is not a gate that opens.
//
// It follows the plans on the bus rather than being restarted, for the same reason the provisioner
// does: a route arriving or leaving is an ordinary event and must not drop the connections of every
// It re-reads on change rather than being restarted, for the same reason the provisioner does:
// a route arriving or leaving is an ordinary event and must not drop the connections of every
// other workload.
package main
@@ -47,6 +48,7 @@ import (
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"net"
@@ -59,6 +61,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/crypto/acme"
@@ -116,8 +119,11 @@ func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy {
}
}
// contribution is one route a consumer asked for: the module, its machine, where that machine is, and what it
// contributes, composed by the mesh (its plan's entry for the route, plans.go).
// what the mesh writes: the contributions file, one entry per consumer.
type given struct {
Given []contribution `json:"given"`
}
type contribution struct {
From string `json:"from"`
Node string `json:"node"`
@@ -173,13 +179,11 @@ type rule struct {
// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
// a proxy's own default refuses them long before the workload is reached.
maxRequestBody int64
// from and node are the consumer that contributed the rule and its machine, which the bus is told (issue 774).
from, node string
}
// table is what the proxy is currently serving, replaced whole whenever the plans change what they give it.
// table is what the proxy is currently serving, replaced whole whenever the file changes.
//
// Replaced rather than merged: the plans are the whole truth about who has a route, so merging
// Replaced rather than merged: the file is the whole truth about who has a route, so merging
// would keep serving a name whose module was unassigned — which is the stale-route fault
// 08-connectivity lists as open, reintroduced one level down.
//
@@ -198,8 +202,6 @@ type table struct {
// machine's address on the private network, as the mesh issued it in this proxy's membership
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
inside sources
// ledger is what this proxy says on the bus about what it serves and refuses (issue 774); nil says nothing.
ledger *ledger
}
// sources is who may be served an internal name: the private network's addresses as the mesh
@@ -287,7 +289,6 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) {
t.to = made
t.public = public
t.mu.Unlock()
t.ledger.serving(made, public)
}
// inOrder puts the rules for one host into the order they are matched in, and does so totally.
@@ -350,7 +351,7 @@ func (t *table) routed(host string) bool {
//
// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity
// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is
// composed only from `to` and `public`, which routesOf already builds correctly — a host never
// composed only from `to` and `public`, which routesFrom already builds correctly — a host never
// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be
// tracked to tell the two apart.
func (t *table) eligibleForInternalACME(host string) bool {
@@ -426,21 +427,48 @@ func main() {
}
func run() error {
path := strings.TrimSpace(os.Getenv("ROUTES"))
if path == "" {
return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing")
}
listen := strings.TrimSpace(os.Getenv("LISTEN"))
if listen == "" {
listen = ":80"
}
// **The seat's plans are the one source of routes** (issue 774, research 072 row 34). Without the bus this proxy
// knows no route, so it refuses to start rather than serving an empty table and saying nothing.
credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE"))
if credential == "" {
return fmt.Errorf("MESH_BROKER_FILE is not set, so this proxy cannot read the plans that say what it routes")
}
held := newTable()
held.ledger = newLedger()
go held.ledger.loop()
go follow(credential, held)
// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
// it an internal name is served to this machine and to nobody else — refused, never opened.
fromBus := &atomic.Bool{}
if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" {
go followMembership(credential, held, fromBus)
} else {
log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+
"internal is served to this machine alone", path)
}
read := func() {
if fromBus.Load() {
return
}
routes, public, err := routesFrom(path)
if err != nil {
// Kept serving what it had. A file being rewritten is momentarily unreadable, and
// dropping every route because one read landed mid-write would turn an ordinary
// event into an outage.
log.Printf("cannot read %s, keeping what is already served: %v", path, err)
return
}
held.set(routes, public)
log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", "))
}
read()
go func() {
for range time.Tick(2 * time.Second) {
read()
}
}()
// TLS is opt-in. A proxy with no `TLS_LISTEN` serves plain HTTP exactly as before — which is
// what an internal-only mesh wants, and what the mesh's own certificate authority already
@@ -459,7 +487,7 @@ func run() error {
"to persist, or every restart orders them again")
}
publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
onlyWhatTheMeshSaid(held), held)
onlyWhatTheMeshSaid(held))
if err != nil {
return err
}
@@ -471,7 +499,7 @@ func run() error {
var internalManager *autocert.Manager
if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
onlyInternalNamesTheMeshSaid(held), held)
onlyInternalNamesTheMeshSaid(held))
if err != nil {
return fmt.Errorf("internal certificate authority: %w", err)
}
@@ -591,7 +619,7 @@ func (p *probedResponse) replayTo(w http.ResponseWriter) {
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
func newManager(cache, directory, bundle string, policy autocert.HostPolicy, held *table) (*autocert.Manager, error) {
func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
client := &acme.Client{DirectoryURL: directory}
var root []byte
if bundle != "" {
@@ -623,7 +651,7 @@ func newManager(cache, directory, bundle string, policy autocert.HostPolicy, hel
// different names because their directories differ.
mine := forThisAuthority(cache, directory, root)
return &autocert.Manager{
Cache: recordingCache{Cache: autocert.DirCache(mine), held: held},
Cache: autocert.DirCache(mine),
Prompt: autocert.AcceptTOS,
HostPolicy: policy,
Client: client,
@@ -672,10 +700,9 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
if internalManager != nil {
fromInternal = internalManager.TLSConfig().GetCertificate
}
pick := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
held.ledger.refused(bareHost(hello.ServerName), true, true)
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
hello.ServerName)
}
@@ -685,28 +712,6 @@ func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Cer
}
return fromPublic(hello)
}
// What the bus is told of a handshake (issue 774): a routed name's certificate, or why it has none; a name not
// routed here, which a browser asking for a site the mesh forgot to route meets before any request is made.
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
cert, err := pick(hello)
name := bareHost(hello.ServerName)
switch {
case name == "":
case !held.routed(name):
held.ledger.refused(name, false, true)
case err != nil:
if !strings.HasPrefix(err.Error(), "no public route for") {
held.ledger.certificateFailed(name, err)
}
case cert != nil:
leaf := cert.Leaf
if leaf == nil && len(cert.Certificate) > 0 {
leaf, _ = x509.ParseCertificate(cert.Certificate[0])
}
held.ledger.handedOut(name, leaf)
}
return cert, err
}
}
// newTable is an empty routing table.
@@ -732,9 +737,6 @@ func handler(held *table) http.Handler {
// this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the
// jail's filter expects it.
log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
if hidden || !held.routed(r.Host) {
held.ledger.refused(bareHost(r.Host), hidden, false)
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound)
if !hidden && held.routed(r.Host) {
@@ -784,14 +786,7 @@ func handler(held *table) http.Handler {
r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
}
// The upstream's answer is what the bus is told of this name (issue 774): its status, or why there was none.
answer := &answerRecorder{ResponseWriter: w}
matched.to.ServeHTTP(answer, r)
status := answer.status
if status == 0 && answer.err == nil && r.Header.Get("Upgrade") != "" {
status = http.StatusSwitchingProtocols
}
held.ledger.answered(bareHost(r.Host), r.URL.Path, status, answer.err)
matched.to.ServeHTTP(w, r)
})
}
@@ -846,13 +841,30 @@ func boolByte(b bool) byte {
return 0
}
// routesOf turns the routes the plans give into host → the rules for that host, and which of those hosts is a public
// name — the second is `name`, ACME-eligible; a host reached only through `internal-name` never appears there.
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
// only through `internal-name` never appears there.
//
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches decides which names the
// mesh composes, so an endpoint that reaches only the private network arrives with an `internal-name` and no `name`.
// That is a whole route, not a malformed one: it is served under its internal name and certified by the internal
// authority. Only a route with neither name has nothing to be served under (novox/hq issue 191).
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches
// decides which names the mesh composes, so an endpoint that reaches only the private network
// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is
// served under its internal name and certified by the internal authority. Only a route with
// neither name has nothing to be served under (novox/hq issue 191).
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, nil, err
}
var said given
if err := json.Unmarshal(raw, &said); err != nil {
return nil, nil, err
}
routes, public := routesOf(said.Given)
return routes, public, nil
}
// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public
// names — the same whether the contributions came in the file or in the membership.
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
out := map[string][]rule{}
public := map[string]bool{}
@@ -943,7 +955,6 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
}
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
}
made.from, made.node = c.From, c.Node
if name != "" {
host := strings.ToLower(name)
@@ -1061,5 +1072,5 @@ func towards(where *url.URL) *httputil.ReverseProxy {
pr.SetURL(where)
pr.Out.Host = pr.In.Host
pr.SetXForwarded()
}, ErrorHandler: proxyFailed}
}}
}
@@ -1,123 +0,0 @@
package main
// The routes this proxy serves, read from the seat mesh-provisioner's plans and nothing else (issue 774; research 072
// row 34, the operator on 2026-10-31: "there is only 1 path, the other is simply deprecated/invalid").
//
// **A route is a provision.** A module that must be reachable requires `route`; the controller composes its plan on
// its node, and the plan's entry for the route names this proxy as its provider and carries what the module
// contributes, composed as the received file used to carry it (its names joined to the node's domains, its endpoint's
// port, its policy), and where the module's machine is on the private network. The proxy follows the plans on the bus
// and serves every entry naming it. There is no file and no second source: a route the plans do not name is not
// served.
import (
"fmt"
"log"
"sort"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provisions"
"route-proxy/internal/routestate"
)
// planned is the routes the seat's plans give this proxy on its node, by plan key, kept so a plan that cannot be read
// leaves what it gave served (a bind missed would read as a route withdrawn).
type planned struct {
node string
mu sync.Mutex
by map[string][]contribution
}
func newPlanned(node string) *planned {
return &planned{node: node, by: map[string][]contribution{}}
}
// take is one change of a plan's key: its value, or nil when the key was deleted. It answers whether what is served
// changed, and says on the log what it could not serve.
func (p *planned) take(key string, value []byte) bool {
p.mu.Lock()
defer p.mu.Unlock()
if value == nil {
_, had := p.by[key]
delete(p.by, key)
return had
}
gave, problems, err := routesOfPlan(p.node, key, value)
if err != nil {
log.Printf("the plan %s is not readable, so what it routes is kept as it was: %v", key, err)
return false
}
for _, problem := range problems {
log.Print(problem)
}
if len(gave) == 0 {
_, had := p.by[key]
delete(p.by, key)
return had
}
p.by[key] = gave
return true
}
// contributions is every route the plans give this proxy, in the order of their plans' keys.
func (p *planned) contributions() []contribution {
p.mu.Lock()
defer p.mu.Unlock()
keys := make([]string, 0, len(p.by))
for k := range p.by {
keys = append(keys, k)
}
sort.Strings(keys)
var out []contribution
for _, k := range keys {
out = append(out, p.by[k]...)
}
return out
}
// routesOfPlan is the routes one plan gives the proxy on node: each contribution of each entry naming this proxy on
// node as the route's provider, reached on its own machine over loopback and on another at the address the plan says.
// A plan of another module, or one routing elsewhere, gives none. Problems are what the plan names and the proxy
// cannot serve, said rather than skipped silently: an entry carrying nothing to serve (a controller before the plans
// carried it), or a consumer elsewhere with no address on the private network.
func routesOfPlan(node, key string, value []byte) ([]contribution, []string, error) {
plan, err := provisions.DecodePlan(value)
if err != nil {
return nil, nil, err
}
if plan.Node+"."+plan.Module != key {
return nil, nil, fmt.Errorf("it is the plan of %s.%s, under the key %s", plan.Node, plan.Module, key)
}
var out []contribution
var problems []string
for _, e := range plan.Plan.Provisions {
if !routesHere(node, e) {
continue
}
if len(e.Given) == 0 {
problems = append(problems, fmt.Sprintf("the plan %s names this proxy for its route and gives nothing "+
"to serve; it is not served", key))
continue
}
at := ""
if plan.Node != node {
if e.ConsumerAt == "" {
problems = append(problems, fmt.Sprintf("the plan %s routes %s on %s here and says no address on "+
"the private network for it; it is not served", key, plan.Module, plan.Node))
continue
}
at = e.ConsumerAt
}
for _, g := range e.Given {
out = append(out, contribution{From: plan.Module, Node: plan.Node, At: at, Values: g.Values})
}
}
return out, problems, nil
}
// routesHere is whether a plan's entry is a route this proxy on node provides.
func routesHere(node string, e catalogue.PlanProvision) bool {
return e.Provision == routestate.Provision && e.Provider == node && e.ProviderModule == routestate.Module
}
@@ -57,7 +57,11 @@ func ask(t *testing.T, proxy, name, path string, auth [2]string) *http.Response
func proxyFor(t *testing.T, routesJSON string) string {
t.Helper()
routes, public, err := routesFrom(routesJSON)
path := filepath.Join(t.TempDir(), "routes.json")
if err := os.WriteFile(path, []byte(routesJSON), 0o644); err != nil {
t.Fatal(err)
}
routes, public, err := routesFrom(path)
if err != nil {
t.Fatal(err)
}
@@ -149,7 +153,11 @@ func TestACredentialInTheDeclarationIsRefusedRatherThanServed(t *testing.T) {
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"$2a$10$abcdefghijklmnopqrstuv"}}]}`,
}
for _, body := range inline {
routes, _, err := routesFrom(body)
path := filepath.Join(t.TempDir(), "routes.json")
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
routes, _, err := routesFrom(path)
if err != nil {
t.Fatal(err)
}
+33 -107
View File
@@ -9,11 +9,9 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strings"
"testing"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/membership"
)
@@ -28,12 +26,12 @@ func behind(t *testing.T, mesh ...string) *table {
at, _ := url.Parse(workload.URL)
host, port, _ := net.SplitHostPort(at.Host)
routes, public, err := routesFrom(fmt.Sprintf(`{"given":[
routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[
{"from":"app","node":"anchor","at":%q,
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}},
{"from":"admin","node":"anchor","at":%q,
"values":{"internal-name":"admin.anchor.internal","port":%s}}
]}`, host, port, host, port))
]}`, host, port, host, port)))
if err != nil {
t.Fatal(err)
}
@@ -167,114 +165,42 @@ func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
}
}
// fakePlans is the seat's plans as a watch hands them: every current key, then each change after Watch returned.
type fakePlans struct {
current map[string]string
handle func(stdio.StateChange) error
}
func (f *fakePlans) Watch(_ string, handle func(stdio.StateChange) error) error {
f.handle = handle
for k, v := range f.current {
if err := handle(stdio.StateChange{Key: k, Op: "put", Value: json.RawMessage(v), Current: true}); err != nil {
return err
}
}
return nil
}
// plan is a plan of the seat mesh-provisioner for module on node, its entries as given.
func plan(node, module, entries string) string {
return fmt.Sprintf(`{"v":1,"node":%q,"module":%q,"plan":{"templates":{"source":"","digest":""},"files":null,`+
`"provisions":[%s]}}`, node, module, entries)
}
// What the plans give is what is served, and nothing else (issue 774, research 072 row 34): each entry naming this
// proxy on its node as the route's provider, every contribution of it, reached over loopback on this machine and at
// the plan's address on another; an entry routing elsewhere, giving nothing or naming no address is not served. A
// change is served as it comes, a plan that cannot be read keeps what it gave, and a deleted one withdraws it. The
// membership says only who is inside: an internal-only name is served to the machines it names.
func TestThePlansAreServedAndTheMembershipSaysWhoIsInside(t *testing.T) {
const gitea = `{"provision":"route","provider":"anchor","provider-module":"route-proxy","consumer-at":"10.10.0.7",` +
`"given":[{"local":"web","values":{"name":"git.example","internal-name":"git.anchor.internal","port":3000}},` +
`{"local":"api","values":{"internal-name":"git-api.anchor.internal","port":3001}}]}`
plans := &fakePlans{current: map[string]string{
"laptop.gitea": plan("laptop", "gitea", gitea+`,{"provision":"postgres-database","provider":"anchor",`+
`"provider-module":"postgres","given":[{"values":{"name":"gitea"}}]}`),
"anchor.blog": plan("anchor", "blog", `{"provision":"route","provider":"anchor","provider-module":"route-proxy",`+
`"consumer-at":"10.10.0.1","given":[{"values":{"name":"blog.example","port":2368}}]}`),
"anchor.other": plan("anchor", "other", `{"provision":"route","provider":"edge","provider-module":"route-proxy",`+
`"given":[{"values":{"name":"other.example","port":80}}]}`),
"anchor.older": plan("anchor", "older", `{"provision":"route","provider":"anchor","provider-module":"route-proxy"}`),
"laptop.lost": plan("laptop", "lost", `{"provision":"route","provider":"anchor","provider-module":"route-proxy",`+
`"given":[{"values":{"name":"lost.example","port":80}}]}`),
}}
// What the mesh issues is what is served: the routes in the membership, internal names to the
// machines it names (novox/hq ADR 0167).
func TestAMembershipIsServedAsIssued(t *testing.T) {
held := newTable()
if err := followPlans(plans, newPlanned("anchor"), held); err != nil {
t.Fatal(err)
}
served := func() map[string]string {
out := map[string]string{}
held.mu.RLock()
defer held.mu.RUnlock()
for host, rules := range held.to {
out[host] = rules[0].target
}
return out
}
want := map[string]string{"git.example": "http://10.10.0.7:3000", "git.anchor.internal": "http://10.10.0.7:3000",
"git-api.anchor.internal": "http://10.10.0.7:3001", "blog.example": "http://127.0.0.1:2368"}
if got := served(); !reflect.DeepEqual(got, want) {
t.Fatalf("serving %v, want %v", got, want)
}
if !held.eligibleForACME("git.example") || held.eligibleForACME("git-api.anchor.internal") {
t.Error("a public name is not certified publicly, or an internal-only one is")
}
for _, row := range []struct {
name, key, value string
served []string
withdrawn []string
}{
{"a route arrives", "anchor.new", plan("anchor", "new", `{"provision":"route","provider":"anchor",`+
`"provider-module":"route-proxy","given":[{"values":{"name":"new.example","port":81}}]}`),
[]string{"new.example"}, nil},
{"an unreadable plan keeps what it gave", "laptop.gitea", `{"v":1,`, []string{"git.example"}, nil},
{"a deleted plan withdraws its routes", "laptop.gitea", "", nil, []string{"git.example", "git-api.anchor.internal"}},
} {
change := stdio.StateChange{Key: row.key, Op: "put", Value: json.RawMessage(row.value)}
if row.value == "" {
change = stdio.StateChange{Key: row.key, Op: "delete"}
}
if err := plans.handle(change); err != nil {
t.Fatal(err)
}
got := served()
for _, host := range row.served {
if _, ok := got[host]; !ok {
t.Errorf("%s: %s is not served (%v)", row.name, host, got)
}
}
for _, host := range row.withdrawn {
if _, ok := got[host]; ok {
t.Errorf("%s: %s is still served", row.name, host)
}
}
}
held = behind(t)
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code != http.StatusNotFound {
t.Errorf("before a membership, a machine of the mesh was served an internal-only name: %d", code)
}
if !applyMembership(membership.Membership{Mesh: []string{"10.10.0.7"}}, held) {
t.Fatal("a membership naming the mesh was not applied")
took := applyMembership(membership.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[
{"from":"admin","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"admin.anchor.internal","port":8080}}]`)},
Mesh: []string{"10.10.0.7"},
}, held)
if !took {
t.Fatal("a membership carrying routes was not applied")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("a machine the membership names was refused the internal-only route")
}
if applyMembership(membership.Membership{Mesh: []string{"not-an-address"}}, held) {
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound {
t.Errorf("a machine the membership does not name was served the internal-only route: %d", code)
}
}
// A membership that says nothing about routes is one from a controller that does not issue them,
// and changes nothing: the file stays the source rather than every route being withdrawn.
func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) {
held := behind(t, "10.10.0.7")
before := held.names()
if applyMembership(membership.Membership{Mesh: []string{"10.10.0.7"}}, held) {
t.Error("a membership without routes was taken as the source of routes")
}
if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") {
t.Errorf("a membership without routes changed what is served: %v, was %v", got, before)
}
if applyMembership(membership.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)},
Mesh: []string{"not-an-address"},
}, held) {
t.Error("a membership whose mesh cannot be read was applied")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("an unreadable membership changed who is inside")
}
}
@@ -2,26 +2,23 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// routesFrom is the table a test's routes make, written as `{"given":[<contribution>…]}`: the contributions the plans
// give the proxy (plans.go), each as routesOfPlan makes it.
func routesFrom(body string) (map[string][]rule, map[string]bool, error) {
var said struct {
Given []contribution `json:"given"`
func write(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "routes.json")
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(body), &said); err != nil {
return nil, nil, err
}
routes, public := routesOf(said.Given)
return routes, public, nil
return path
}
// plain is the table an ordinary set of routes makes: one host, one target, no policy.
@@ -54,9 +51,9 @@ func targetOf(routes map[string][]rule, host string) string {
// A route is a grant: the consumer supplies a target, and where that machine is comes from the
// mesh rather than from a naming convention the proxy has to know.
func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
routes, _, err := routesFrom(`{"contributions":1,"requirement":"route","given":[
routes, _, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[
{"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -71,10 +68,10 @@ func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
// target — the same convenience a predecessor proxy gave for reaching a service over the VPN
// without a public TLS round trip.
func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -97,10 +94,10 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing —
// skipping it left every internal-only module unreachable by name (novox/hq issue 191).
func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -124,11 +121,24 @@ func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
}
}
// A route with neither name has nothing to be served under, and is still skipped.
func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 || len(public) != 0 {
t.Errorf("a route that named nothing was served: %v %v", routes, public)
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","values":{"name":"app.example","port":8080}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -140,9 +150,9 @@ func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
// A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the
// only thing that works when there is no private network.
func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","values":{"name":"app.example","port":9000}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -151,13 +161,28 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
}
}
// Skipped rather than served wrongly. A route with no port would proxy to :0.
func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 1 || targetOf(routes, "fine.example") == "" {
t.Fatalf("an unusable contribution was served: %v", routes)
}
}
// A route may name a target reached over https, for a backend that terminates its own TLS — the
// shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise.
func TestARouteMayTargetHttps(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"mail","node":"anchor","at":"anchor.internal",
"values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -169,6 +194,19 @@ func TestARouteMayTargetHttps(t *testing.T) {
}
}
// A scheme that is neither http nor https is refused rather than guessed at.
func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 {
t.Fatalf("a route with an unusable scheme was served: %v", routes)
}
}
// End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still
// reached when the route declared `insecure`, and the response comes back through unmodified.
func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) {
@@ -322,10 +360,10 @@ func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) {
// public CA can validate a private name, and asking anyway would only spend the account's rate
// limit on an order that can never succeed.
func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -344,10 +382,10 @@ func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) {
// A certificate is asked of the *internal* authority only for a name that is routed here and is
// not a route's own public name — the internal-network alias, never the route it accompanies.
func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) {
routes, public, err := routesFrom(`{"given":[
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -388,10 +426,10 @@ func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) {
// default refuses them long before the workload is reached. It is configuration beside `insecure`,
// not a fifth policy — novox/hq ADR 0108 closed that set at four.
func TestARouteMayLimitTheBodyItCarries(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"anchor.internal",
"values":{"name":"images.example","port":5000,"max-request-body":21474836480}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -406,9 +444,9 @@ func TestARouteMayLimitTheBodyItCarries(t *testing.T) {
// Saying nothing leaves the route unlimited, which is what every route already got.
func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
routes, _, err := routesFrom(`{"given":[
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"name":"app.example","port":8080}}
]}`)
]}`))
if err != nil {
t.Fatal(err)
}
@@ -417,6 +455,23 @@ func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
}
}
// A limit that is not a whole positive number of bytes takes the route with it. Serving it without
// the limit would carry exactly what the module said not to carry, and report success doing it.
func TestARouteWithAnUnusableLimitIsSkipped(t *testing.T) {
for _, asked := range []string{`"lots"`, `-1`, `0`, `1.5`} {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"anchor.internal",
"values":{"name":"images.example","port":5000,"max-request-body":`+asked+`}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes["images.example"]) != 0 {
t.Errorf("a route asking for a max-request-body of %s was served anyway: %v", asked, routes)
}
}
}
// A request larger than the route carries is refused by the proxy, with the limit named, and the
// workload never sees it. A request within it is proxied normally.
func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
@@ -430,10 +485,10 @@ func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
at := strings.TrimPrefix(workload.URL, "http://")
host, port, _ := strings.Cut(at, ":")
routes, _, err := routesFrom(`{"given":[
{"from":"registry","node":"anchor","at":"` + host + `",
"values":{"name":"images.example","port":` + port + `,"max-request-body":8}}
]}`)
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"`+host+`",
"values":{"name":"images.example","port":`+port+`,"max-request-body":8}}
]}`))
if err != nil {
t.Fatal(err)
}
@@ -477,32 +532,3 @@ func post(url, host, body string) (*http.Response, error) {
asked.Header.Set("Content-Type", "application/octet-stream")
return http.DefaultClient.Do(asked)
}
// A contribution that cannot be served is skipped rather than served wrongly: one that names nothing to be served
// under; one with no port, which would proxy to :0; a scheme that is neither http nor https, refused rather than
// guessed at; and a body limit that is not a whole positive number of bytes, since serving it without the limit would
// carry exactly what the module said not to carry, and report success doing it.
func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) {
for _, row := range []struct{ why, given, served string }{
{"neither name", `{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}`, ""},
{"no port", `{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}`, "fine.example"},
{"an unknown scheme", `{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}`, ""},
{"a limit in words", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":"lots"}}`, ""},
{"a negative limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":-1}}`, ""},
{"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""},
{"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""},
} {
routes, public, err := routesFrom(`{"given":[` + row.given + `]}`)
if err != nil {
t.Fatal(err)
}
switch {
case row.served == "" && (len(routes) != 0 || len(public) != 0):
t.Errorf("%s: an unusable contribution was served: %v %v", row.why, routes, public)
case row.served != "" && (len(routes) != 1 || targetOf(routes, row.served) == ""):
t.Errorf("%s: an unusable contribution was served beside the usable one: %v", row.why, routes)
}
}
}
-77
View File
@@ -1,77 +0,0 @@
// Package contract is route-proxy's contract for the provision `route`: what a consumer asks a route with, what the
// proxy keeps on the bus about what it serves, the events it says, and its tools' arguments (issue 774; research 072
// row 34; the operator, 2026-10-31: "the route-proxy should define its own contract for the seat (with the message
// contracts of course) which other modules can use").
//
// **Published by the repository that holds route-proxy, as its own Go module**, versioned by tags of the form
// `modules/route-proxy/contract/vX.Y.Z`, the subdirectory form Go's module system expects, the same way mesh-sdk is
// published (`go/vX.Y.Z`). A module that asks a route, reads what the proxy serves or hears its events imports this
// package; nothing imports the repository. Standard library only, so a consumer takes nothing else with it.
//
// **How a route travels.** A module that must be reachable requires `route` and says, in its manifest's
// `contributes.route`, one Request, or several by local name; Schema is that object's JSON schema. The controller
// composes each (its label joined to the node's domains into `name` and `internal-name`, its endpoint's port into
// `port`) and the seat mesh-provisioner carries the result in the consumer's plan entry as an opaque payload
// (mesh-sdk catalogue.PlanProvision.Contributed, in the generic catalogue.ContributedForm). Neither the seat nor its
// client knows a field of a route: route-proxy decodes each contribution with DecodeRequest, and refuses, saying why
// (EventRefused), what this contract does not admit.
//
// **Two name spaces, two authorities** (novox/hq ADR 0056, ADR 0138). `name` is public: served to anyone and
// certified by the public ACME directory (Let's Encrypt). `internal-name` is served only inside the private network
// and certified by the mesh's own authority. A route carries either, or both.
package contract
// Module is the module that provides a route, and Provision what it provides: what a plan's entry names when
// route-proxy is a route's provider.
const (
Module = "route-proxy"
Provision = "route"
)
// Version is the version of the Request form this package reads and writes.
const Version = 1
// The events route-proxy says (its manifest's `emits`), on mesh.mod.route-proxy.event.<type>, each with an Event.
const (
// EventServed: a name is now served by this proxy (it was not, or it is now served from other rules).
EventServed = "route.served"
// EventWithdrawn: a name this proxy served is served no more, because no plan gives it any longer.
EventWithdrawn = "route.withdrawn"
// EventRefused: a plan gives this proxy a route it does not serve, and Why says why.
EventRefused = "route.refused"
)
// Event is the body of every event route-proxy says: the proxy's machine, the name (empty for a refusal that named
// none), how far it reaches, and who asked it: the consumer module, its machine and the local name of its
// contribution. Why is set on a refusal alone.
type Event struct {
Proxy string `json:"proxy"`
Name string `json:"name,omitempty"`
Reach string `json:"reach,omitempty"`
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
Local string `json:"local,omitempty"`
Why string `json:"why,omitempty"`
}
// The tools route-proxy serves (its manifest's `tools`), each answering from its states RoutesState and UnroutedState.
const (
// ToolRoutes lists every name served, with RoutesArgs.
ToolRoutes = "route_proxy_routes"
// ToolRoute is one name in full, with RouteArgs.
ToolRoute = "route_proxy_route"
// ToolUnrouted lists the names asked for and not served, with RoutesArgs.
ToolUnrouted = "route_proxy_unrouted"
)
// RoutesArgs are ToolRoutes' and ToolUnrouted's arguments: Node narrows to the proxy on one machine, every machine
// when empty.
type RoutesArgs struct {
Node string `json:"node,omitempty"`
}
// RouteArgs are ToolRoute's arguments: the name, such as app.example.org, and optionally the machine whose proxy.
type RouteArgs struct {
Name string `json:"name"`
Node string `json:"node,omitempty"`
}
@@ -1,85 +0,0 @@
package contract
import (
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
)
func limit(n int64) *int64 { return &n }
// A route is read strictly and whole (issue 774): what the contract names is admitted, what it does not, or a route
// it cannot serve, is refused naming why; and the published schema names exactly the fields the decoder reads.
func TestARouteIsReadStrictly(t *testing.T) {
for _, c := range []struct {
raw, refused string
want Request
}{
{raw: `{"name":"app.example.org","internal-name":"app.anchor.internal","label":"app","endpoint":"web","port":8080}`,
want: Request{Name: "app.example.org", InternalName: "app.anchor.internal", Label: "app", Endpoint: "web", Port: 8080}},
{raw: `{"internal-name":"app.anchor.internal","port":443,"scheme":"https","insecure":true,"max-request-body":1048576}`,
want: Request{InternalName: "app.anchor.internal", Port: 443, Scheme: "https", Insecure: true, MaxRequestBody: limit(1048576)}},
{raw: `{"name":"git.example.org","path":"/api/internal","deny":true,"priority":100000}`,
want: Request{Name: "git.example.org", Path: "/api/internal", Deny: true, Priority: 100000}},
{raw: `{"name":"old.example.org","redirect":"https://new.example.org"}`,
want: Request{Name: "old.example.org", Redirect: "https://new.example.org"}},
{raw: `{"name":"admin.example.org","port":80,"auth":"/run/secrets/admin"}`,
want: Request{Name: "admin.example.org", Port: 80, Auth: "/run/secrets/admin"}},
{raw: `{"label":"app","port":8080}`, refused: "names nothing"},
{raw: `{"name":"app.example.org"}`, refused: "gives no port"},
{raw: `{"name":"app.example.org","port":70000}`, refused: "not a port"},
{raw: `{"name":"app.example.org","port":80.5}`, refused: "not a route"},
{raw: `{"name":"app.example.org","port":"80"}`, refused: "not a route"},
{raw: `{"name":"app.example.org","port":80,"scheme":"ftp"}`, refused: "neither http nor https"},
{raw: `{"name":"app.example.org","port":80,"max-request-body":-1}`, refused: "max-request-body"},
{raw: `{"name":"app.example.org","port":80,"max-request-body":0}`, refused: "max-request-body"},
{raw: `{"name":"app.example.org","port":80,"auth":"admin:$2y$05$abc"}`, refused: "credential"},
{raw: `{"name":"app.example.org","port":80,"host":"x"}`, refused: "unknown field \"host\""},
{raw: `{"name":"app.example.org","port":80} {}`, refused: "more than one"},
} {
got, err := DecodeRequest([]byte(c.raw))
if c.refused != "" {
if err == nil || !strings.Contains(err.Error(), c.refused) {
t.Errorf("DecodeRequest(%s) = %+v, %v; want refused for %q", c.raw, got, err, c.refused)
}
continue
}
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("DecodeRequest(%s) = %+v, %v; want %+v", c.raw, got, err, c.want)
continue
}
// What it encodes, it reads back the same.
if raw, err := got.Encode(); err != nil {
t.Errorf("Encode(%+v): %v", got, err)
} else if again, err := DecodeRequest(raw); err != nil || !reflect.DeepEqual(again, got) {
t.Errorf("Encode then DecodeRequest of %+v = %+v, %v", got, again, err)
}
}
if (Request{Path: "api"}).Prefix() != "/api" || (Request{}).Prefix() != "" || (Request{Scheme: " HTTPS"}).SchemeOrDefault() != "https" ||
(Request{}).SchemeOrDefault() != "http" || (Request{Redirect: "x"}).Forwards() {
t.Error("Prefix, SchemeOrDefault or Forwards says otherwise than the contract")
}
var schema struct {
Properties map[string]json.RawMessage `json:"properties"`
AdditionalProperties bool `json:"additionalProperties"`
}
if err := json.Unmarshal(Schema, &schema); err != nil || schema.AdditionalProperties {
t.Fatalf("the schema is not a closed object: %v", err)
}
var tags []string
for f := range reflect.TypeFor[Request]().Fields() {
tags = append(tags, strings.Split(f.Tag.Get("json"), ",")[0])
}
var said []string
for name := range schema.Properties {
said = append(said, name)
}
slices.Sort(tags)
slices.Sort(said)
if !slices.Equal(tags, said) {
t.Fatalf("the schema names %v, the decoder reads %v", said, tags)
}
}
-3
View File
@@ -1,3 +0,0 @@
module git.novox.be/novox/mesh-controller/modules/route-proxy/contract
go 1.26.0
-128
View File
@@ -1,128 +0,0 @@
package contract
import (
"bytes"
_ "embed"
"encoding/json"
"errors"
"fmt"
"strings"
)
// Request is one route a consumer asks for, as its manifest's `contributes.route` says it and the controller composes
// it: the object Schema describes. Every field is optional to the decoder; Validate says which a route needs.
type Request struct {
// Name is the public name, served to anyone and certified by the public ACME directory; InternalName the name on
// the private network, served inside it alone and certified by the mesh's own authority. The controller composes
// both from Label and the node's domains, as far as the endpoint reaches (novox/hq ADR 0138); a manifest writes a
// label, never a domain.
Name string `json:"name,omitempty"`
InternalName string `json:"internal-name,omitempty"`
Label string `json:"label,omitempty"`
// Endpoint names the manifest's endpoint the route serves; the controller fills Port from it.
Endpoint string `json:"endpoint,omitempty"`
Port int `json:"port,omitempty"`
// Path is the path prefix the rule is scoped to, every path when empty; Priority which rule wins where two match,
// higher first.
Path string `json:"path,omitempty"`
Priority int `json:"priority,omitempty"`
// Scheme is how the consumer is reached, http (the default) or https; Insecure skips verifying its certificate,
// for a backend that terminates TLS with its own.
Scheme string `json:"scheme,omitempty"`
Insecure bool `json:"insecure,omitempty"`
// MaxRequestBody is the largest body, in bytes, the route carries; no limit when absent. Said, it is whole and
// positive: a limit the proxy cannot keep is a route it does not serve.
MaxRequestBody *int64 `json:"max-request-body,omitempty"`
// The policy of the rule, a closed set (novox/hq ADR 0108): Deny refuses every request, Redirect answers with a
// permanent redirect to that address, and Auth names the file of the secret a request must present, never the
// secret itself.
Deny bool `json:"deny,omitempty"`
Redirect string `json:"redirect,omitempty"`
Auth string `json:"auth,omitempty"`
}
// Schema is Request's JSON schema, published with this package for what checks a manifest's `contributes.route`.
//
//go:embed route.schema.json
var Schema []byte
// DecodeRequest reads one contribution's object strictly: no field this contract does not name, one document, and
// a route Validate admits. What it refuses, it refuses whole, naming why: a route is never served half.
func DecodeRequest(raw []byte) (Request, error) {
var r Request
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&r); err != nil {
return Request{}, fmt.Errorf("not a route this contract reads: %w", err)
}
if dec.More() {
return Request{}, errors.New("not a route this contract reads: more than one document")
}
if err := r.Validate(); err != nil {
return Request{}, err
}
return r, nil
}
// Encode is r as a contribution's object, refused where DecodeRequest would refuse it.
func (r Request) Encode() ([]byte, error) {
if err := r.Validate(); err != nil {
return nil, err
}
return json.Marshal(r)
}
// Validate says what makes r no route: no name of either kind; a port outside 1-65535, or none where the rule
// forwards (neither Deny nor Redirect); a scheme other than http or https; a body limit that is not positive; and a
// credential written where a secret's file is to be named (novox/hq ADR 0108: refused rather than tolerated).
func (r Request) Validate() error {
if strings.TrimSpace(r.Name) == "" && strings.TrimSpace(r.InternalName) == "" {
return errors.New("the route names nothing to be served under: neither name nor internal-name")
}
if r.Port < 0 || r.Port > 65535 {
return fmt.Errorf("the route's port %d is not a port", r.Port)
}
if r.Forwards() && r.Port == 0 {
return errors.New("the route forwards and gives no port")
}
switch strings.ToLower(strings.TrimSpace(r.Scheme)) {
case "", "http", "https":
default:
return fmt.Errorf("the route's scheme %q is neither http nor https", r.Scheme)
}
if r.MaxRequestBody != nil && *r.MaxRequestBody <= 0 {
return fmt.Errorf("the route's max-request-body %d is not a positive number of bytes", *r.MaxRequestBody)
}
if LooksLikeACredential(r.Auth) {
return errors.New("the route's auth holds a credential rather than the name of a secret's file (novox/hq ADR 0108)")
}
return nil
}
// Forwards is whether the rule sends a request on to the consumer, rather than answering it itself.
func (r Request) Forwards() bool { return !r.Deny && r.Redirect == "" }
// Prefix is the path prefix the rule is scoped to, beginning with a slash, or "" for every path.
func (r Request) Prefix() string {
p := strings.TrimSpace(r.Path)
if p != "" && !strings.HasPrefix(p, "/") {
p = "/" + p
}
return p
}
// SchemeOrDefault is how the consumer is reached: the scheme, lower-cased, http when none is said.
func (r Request) SchemeOrDefault() string {
if s := strings.ToLower(strings.TrimSpace(r.Scheme)); s != "" {
return s
}
return "http"
}
// LooksLikeACredential is the check that keeps a secret out of a declaration: a value holding a `:` (the htpasswd
// separator) or opening with a bcrypt identifier is a credential, not a path, and no path the mesh writes needs
// either. It errs towards refusing.
func LooksLikeACredential(v string) bool {
v = strings.TrimSpace(v)
return strings.Contains(v, ":") || strings.HasPrefix(v, "$2")
}
@@ -1,28 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://git.novox.be/novox/mesh-controller/modules/route-proxy/contract/route.schema.json",
"title": "route",
"description": "One route a consumer asks route-proxy for, as a manifest's contributes.route says it (one, or several by local name) and the controller composes it. Version 1 of route-proxy's contract.",
"type": "object",
"additionalProperties": false,
"anyOf": [
{"required": ["name"]},
{"required": ["internal-name"]},
{"required": ["label"]}
],
"properties": {
"name": {"type": "string", "description": "the public name, served to anyone and certified by the public ACME directory; composed from label"},
"internal-name": {"type": "string", "description": "the name on the private network, served inside it alone and certified by the mesh's own authority; composed from label"},
"label": {"type": "string", "description": "the subdomain the module chooses; joined to the node's domains by the controller"},
"endpoint": {"type": "string", "description": "the manifest's endpoint the route serves; the controller fills port from it"},
"port": {"type": "integer", "minimum": 1, "maximum": 65535},
"path": {"type": "string", "description": "the path prefix the rule is scoped to; every path when absent"},
"priority": {"type": "integer", "description": "which rule wins where two match, higher first"},
"scheme": {"enum": ["http", "https"], "description": "how the consumer is reached; http when absent"},
"insecure": {"type": "boolean", "description": "skip verifying the consumer's certificate over https"},
"max-request-body": {"type": "integer", "minimum": 1, "description": "the largest body, in bytes, the route carries"},
"deny": {"type": "boolean", "description": "refuse every request matching the rule"},
"redirect": {"type": "string", "description": "answer with a permanent redirect to this address"},
"auth": {"type": "string", "description": "the file of the secret a request must present, never the secret itself"}
}
}
-152
View File
@@ -1,152 +0,0 @@
package contract
// What the web front keeps on the bus about the names it serves, and about the names it was asked for and does not
// serve (issue 774). The proxy writes it, on change, in its states `routes` and `unrouted`, announced readable; the
// module's tools, and any module that reads them, read it from there, so the question "is this site reachable, and
// is its certificate good" is answered from the bus rather than with curl.
//
// A key is `<node>.<name>`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds
// no dot, so the first dot splits them.
import (
"net"
"regexp"
"strings"
"time"
)
// The states route-proxy keeps, by their local names (the buckets are route-proxy_<name>).
const (
// RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got.
RoutesState = "routes"
// UnroutedState holds one key per name the proxy answered "no route for" with, and when it was last asked.
UnroutedState = "unrouted"
// BoundState is the provider helper's bucket of consumer entries (mesh-sdk go/provider).
BoundState = "bound"
)
// The words a route's reach and a certificate's authority are said in.
const (
ReachPublic = "public"
ReachInternal = "internal"
)
// The words a certificate's outcome is said in.
const (
// Issued: the authority issued it while this proxy ran, at RenewedAt.
Issued = "issued"
// Cached: this proxy served it from what it kept, and saw no issuance of it since it started.
Cached = "cached"
// Failed: the last attempt to have one failed, Error says why.
Failed = "failed"
)
// Target is one rule a name is served by: what a request matching it reaches, or what it is answered with.
type Target struct {
// Path is the path prefix the rule is scoped to; empty is every path.
Path string `json:"path,omitempty"`
// Module and Node are the consumer that contributed the route and the machine it runs on.
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
// Target is where the request is forwarded, scheme://address:port; empty for a rule that answers itself.
Target string `json:"target,omitempty"`
Deny bool `json:"deny,omitempty"`
Redirect string `json:"redirect,omitempty"`
// Auth says the rule asks for credentials; Sealed says why it cannot check them, and so refuses everything.
Auth bool `json:"auth,omitempty"`
Sealed string `json:"sealed,omitempty"`
}
// Certificate is what this proxy knows of a name's certificate.
type Certificate struct {
// Authority is which issuer certifies the name: public (the ACME directory the module names) or internal (the
// mesh's own).
Authority string `json:"authority"`
Issuer string `json:"issuer,omitempty"`
NotAfter *time.Time `json:"not_after,omitempty"`
// RenewedAt is when an authority last issued it to this proxy.
RenewedAt *time.Time `json:"renewed_at,omitempty"`
// Outcome is issued, cached or failed; empty until a handshake or an issuance said anything.
Outcome string `json:"outcome,omitempty"`
Error string `json:"error,omitempty"`
At *time.Time `json:"at,omitempty"`
}
// Answer is the last answer the proxy got from upstream for a name: the status it carried back, or the error that
// stood in for one (a 502).
type Answer struct {
Status int `json:"status"`
At time.Time `json:"at"`
Error string `json:"error,omitempty"`
Path string `json:"path,omitempty"`
}
// Route is one served name, the value of a key of RoutesState.
type Route struct {
Name string `json:"name"`
// Proxy is the machine whose proxy serves it.
Proxy string `json:"proxy"`
// Reach is public (a route's own name) or internal (only its internal-name, served inside the private network).
Reach string `json:"reach"`
Targets []Target `json:"targets"`
Certificate Certificate `json:"certificate"`
LastAnswer *Answer `json:"last_answer,omitempty"`
}
// Unrouted is one name the proxy answered "no route for", the value of a key of UnroutedState.
type Unrouted struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
// AskedAt is when the name was last asked for. Who asked is not kept here: the journal keeps it, for the
// machine's intrusion prevention.
AskedAt time.Time `json:"asked_at"`
// Times is how often it was asked since the proxy last started.
Times int `json:"times"`
// InternalOnly says the name is served, but only inside the private network, and was asked from outside.
InternalOnly bool `json:"internal_only,omitempty"`
// Handshake says it was asked in a TLS handshake (the name a client sent), not in an HTTP request.
Handshake bool `json:"handshake,omitempty"`
}
// OtherName is the name of the one key per proxy that counts the refusals not kept by name: names under no domain
// the proxy serves (what a scan sends), and names past the limit of new names a minute. Its key is
// `<node>._other`; no host name begins with an underscore.
const OtherName = "_other"
// Other is the value of a proxy's OtherName key.
type Other struct {
Proxy string `json:"proxy"`
// Minute is the last whole minute counted, and LastMinute how many refusals it held.
Minute time.Time `json:"minute"`
LastMinute int `json:"last_minute"`
// SinceStart is every refusal not kept by name since the proxy started.
SinceStart int `json:"since_start"`
}
// Registrable is the domain a name is registered under, as its last two labels; "" for an address or a single
// label. Two labels is wrong for a public suffix of two (co.uk); it only decides which refused names are kept by name.
func Registrable(name string) string {
if net.ParseIP(strings.Trim(name, "[]")) != nil {
return ""
}
labels := strings.Split(name, ".")
if len(labels) < 2 {
return ""
}
return strings.Join(labels[len(labels)-2:], ".")
}
var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`)
// Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request
// is whatever a client sent; one no key could hold is no site anybody runs.
func Keyable(name string) bool { return keyable.MatchString(name) && !strings.Contains(name, "..") }
// Key is the key of name served or asked at node's proxy.
func Key(node, name string) string { return node + "." + name }
// Split is a key's node and name.
func Split(key string) (node, name string) {
node, name, _ = strings.Cut(key, ".")
return node, name
}
+1 -1
View File
@@ -3,7 +3,7 @@ module route-proxy
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68 h1:2rMJX0RMIBaN5NDIaEZVwbj5MFism1O3bRayEz+WlOs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,158 +0,0 @@
// Package routestate is what the web front keeps on the bus about the names it serves, and about the names it was asked
// for and does not serve (issue 774): the shape both halves of route-proxy agree on. The proxy writes it, on change,
// in its states `routes` and `unrouted`, announced readable; the module's tools read it from there, so the question
// "is this site reachable, and is its certificate good" is answered from the bus rather than with curl.
//
// A key is `<node>.<name>`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds
// no dot, so the first dot splits them.
package routestate
import (
"net"
"regexp"
"strings"
"time"
)
// Module is route-proxy, and Provision what it provides: what a plan's entry names when this proxy is a route's provider
// (both halves read the seat mesh-provisioner's plans for it, issue 774).
const (
Module = "route-proxy"
Provision = "route"
)
// The states route-proxy keeps, by their local names (the buckets are route-proxy_<name>).
const (
// RoutesState holds one key per served name: where it forwards, its certificate, the last answer it got.
RoutesState = "routes"
// UnroutedState holds one key per name the proxy answered "no route for" with, and when it was last asked.
UnroutedState = "unrouted"
// BoundState is the provider helper's bucket of consumer entries (mesh-sdk go/provider).
BoundState = "bound"
)
// The words a route's reach and a certificate's authority are said in.
const (
ReachPublic = "public"
ReachInternal = "internal"
)
// The words a certificate's outcome is said in.
const (
// Issued: the authority issued it while this proxy ran, at RenewedAt.
Issued = "issued"
// Cached: this proxy served it from what it kept, and saw no issuance of it since it started.
Cached = "cached"
// Failed: the last attempt to have one failed, Error says why.
Failed = "failed"
)
// Target is one rule a name is served by: what a request matching it reaches, or what it is answered with.
type Target struct {
// Path is the path prefix the rule is scoped to; empty is every path.
Path string `json:"path,omitempty"`
// Module and Node are the consumer that contributed the route and the machine it runs on.
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
// Target is where the request is forwarded, scheme://address:port; empty for a rule that answers itself.
Target string `json:"target,omitempty"`
Deny bool `json:"deny,omitempty"`
Redirect string `json:"redirect,omitempty"`
// Auth says the rule asks for credentials; Sealed says why it cannot check them, and so refuses everything.
Auth bool `json:"auth,omitempty"`
Sealed string `json:"sealed,omitempty"`
}
// Certificate is what this proxy knows of a name's certificate.
type Certificate struct {
// Authority is which issuer certifies the name: public (the ACME directory the module names) or internal (the
// mesh's own).
Authority string `json:"authority"`
Issuer string `json:"issuer,omitempty"`
NotAfter *time.Time `json:"not_after,omitempty"`
// RenewedAt is when an authority last issued it to this proxy.
RenewedAt *time.Time `json:"renewed_at,omitempty"`
// Outcome is issued, cached or failed; empty until a handshake or an issuance said anything.
Outcome string `json:"outcome,omitempty"`
Error string `json:"error,omitempty"`
At *time.Time `json:"at,omitempty"`
}
// Answer is the last answer the proxy got from upstream for a name: the status it carried back, or the error that
// stood in for one (a 502).
type Answer struct {
Status int `json:"status"`
At time.Time `json:"at"`
Error string `json:"error,omitempty"`
Path string `json:"path,omitempty"`
}
// Route is one served name, the value of a key of RoutesState.
type Route struct {
Name string `json:"name"`
// Proxy is the machine whose proxy serves it.
Proxy string `json:"proxy"`
// Reach is public (a route's own name) or internal (only its internal-name, served inside the private network).
Reach string `json:"reach"`
Targets []Target `json:"targets"`
Certificate Certificate `json:"certificate"`
LastAnswer *Answer `json:"last_answer,omitempty"`
}
// Unrouted is one name the proxy answered "no route for", the value of a key of UnroutedState.
type Unrouted struct {
Name string `json:"name"`
Proxy string `json:"proxy"`
// AskedAt is when the name was last asked for. Who asked is not kept here: the journal keeps it, for the
// machine's intrusion prevention.
AskedAt time.Time `json:"asked_at"`
// Times is how often it was asked since the proxy last started.
Times int `json:"times"`
// InternalOnly says the name is served, but only inside the private network, and was asked from outside.
InternalOnly bool `json:"internal_only,omitempty"`
// Handshake says it was asked in a TLS handshake (the name a client sent), not in an HTTP request.
Handshake bool `json:"handshake,omitempty"`
}
// OtherName is the name of the one key per proxy that counts the refusals not kept by name: names under no domain
// the proxy serves (what a scan sends), and names past the limit of new names a minute. Its key is
// `<node>._other`; no host name begins with an underscore.
const OtherName = "_other"
// Other is the value of a proxy's OtherName key.
type Other struct {
Proxy string `json:"proxy"`
// Minute is the last whole minute counted, and LastMinute how many refusals it held.
Minute time.Time `json:"minute"`
LastMinute int `json:"last_minute"`
// SinceStart is every refusal not kept by name since the proxy started.
SinceStart int `json:"since_start"`
}
// Registrable is the domain a name is registered under, as its last two labels; "" for an address or a single
// label. Two labels is wrong for a public suffix of two (co.uk); it only decides which refused names are kept by name.
func Registrable(name string) string {
if net.ParseIP(strings.Trim(name, "[]")) != nil {
return ""
}
labels := strings.Split(name, ".")
if len(labels) < 2 {
return ""
}
return strings.Join(labels[len(labels)-2:], ".")
}
var keyable = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_.-]{0,251}[a-z0-9_])?$`)
// Keyable says whether a name can be a key's name: what a host name or an address is. A name asked for in a request
// is whatever a client sent; one no key could hold is no site anybody runs.
func Keyable(name string) bool { return keyable.MatchString(name) && !strings.Contains(name, "..") }
// Key is the key of name served or asked at node's proxy.
func Key(node, name string) string { return node + "." + name }
// Split is a key's node and name.
func Split(key string) (node, name string) {
node, name, _ = strings.Cut(key, ".")
return node, name
}
+13 -24
View File
@@ -16,6 +16,9 @@
"serves": {
"route": {}
},
"receives": {
"route": "${dir:routes-dir}/mesh.json"
},
"requires": [
"internal-acme-ca"
],
@@ -70,6 +73,12 @@
"mode": "0700",
"place": "mesh"
},
{
"id": "routes-dir",
"type": "directory",
"path": "/var/lib/route-proxy/routes",
"mode": "0700"
},
{
"id": "acme-cache",
"type": "directory",
@@ -158,12 +167,14 @@
"${dir:state}/internal-acme.env"
],
"volumes": [
"${dir:routes-dir}:/routes:ro",
"${dir:acme-cache}:/acme",
"${dir:ca-dir}:/ca:ro",
"${dir:mesh-state}/broker:/run/secrets/broker:ro",
"${dir:state}/route-proxy:/opt/mesh:ro"
],
"env": {
"ROUTES": "/routes/mesh.json",
"LISTEN": ":80",
"TLS_LISTEN": ":443",
"ACME_CACHE": "/acme",
@@ -233,31 +244,9 @@
}
],
"state": [
"bound",
"routes",
{
"name": "unrouted",
"ttl-seconds": 604800
}
"bound"
],
"tools": [
"route_proxy_routes",
"route_proxy_route",
"route_proxy_unrouted"
],
"replaces": {
"route_proxy_routes": [
"curl -I"
],
"route_proxy_route": [
"openssl s_client -connect"
],
"route_proxy_unrouted": [
"docker logs route-proxy"
]
},
"reads": [
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
"mesh-controller.assignments"
]
}
+2 -2
View File
@@ -7,9 +7,9 @@ The copy carries the catalogue's private details: domains, first names in module
private addresses. That is acceptable while mesh-controller is private; if it ever goes public, this copy is
a second place to clean besides the catalogue itself.
mesh-catalog f3a57dcc56fb6c198095cafd11d0decff9f720c4 modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
mesh-catalog 3a1017f5594217092a654dd1e2d72226766bc16a modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock
mesh-sdk 2da0de75c3f0087cff9c2f5ef789ca6dc3c59045 conformance/events/module-event.json
mesh-sdk 10b6e202320f38301c67a6822b0734e0b870ea30 conformance/events/module-event.json
To move them, from this repository's root, with the two repositories checked out beside it:
@@ -1 +0,0 @@
${seat:mesh-artifact-store:reach}
@@ -1,52 +1,26 @@
{
"module": "artifact-store-tools",
"version": "1",
"claims": [
{
"name": "mesh-artifact-sweep",
"scope": "mesh",
"serves": [
"artifacts",
"held",
"collect",
"mirrors",
"images"
]
}
],
"requires": [
"artifact-store"
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"seat:mesh-controller.artifacts",
"seat:mesh-controller.collect",
"mesh-catalog.catalog_modules",
"mesh-catalog.catalog_module"
],
"reads": [
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-builder.build-log",
"mesh-delivery-indexer.manifests",
"node-engine.have",
"mesh-controller.lease"
],
"state": [
"findings"
],
"tools": [
"artifact_store_repositories",
"artifact_store_usage",
"artifact_store_references",
"artifact_store_collect"
],
"config": {
"older-store-names": {
"type": "string",
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
"default": "none"
}
},
"config": {
"older-store-names": {
"type": "string",
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
"default": "none"
}
},
"resources": [
{
"id": "state",
@@ -62,14 +36,6 @@
"mode": "0600",
"trusted": false,
"template": "files/store-names"
},
{
"id": "store-address",
"type": "file",
"path": "${dir:state}/store-address",
"mode": "0600",
"trusted": false,
"template": "files/store-address"
}
],
"build": {
@@ -86,8 +52,7 @@
],
"env": {
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names",
"MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address"
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names"
}
},
{
@@ -8,10 +8,7 @@
"provides": [
{
"name": "rootless-image-build",
"reach": "machine",
"credential": {
"form": "none"
}
"reach": "machine"
}
],
"resources": [
@@ -7,9 +7,7 @@
"disk_filesystems",
"disk_folders",
"disk_devices",
"cpu_top",
"disk_readings",
"disk_measure"
"cpu_top"
],
"build": {
"artifacts": [
@@ -25,16 +23,5 @@
]
}
]
},
"state": [
"readings"
],
"reads": [
"mesh-config.resolved",
"mesh-delivery-assigner.placement",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"node-engine.have"
]
}
}
@@ -4,10 +4,7 @@
"provides": [
{
"name": "artifact-store",
"scope": "mesh",
"credential": {
"form": "none"
}
"scope": "mesh"
}
],
"claims": [
@@ -1,15 +1,14 @@
# Written by the mesh (module dnsmasq, novox/hq issue 348). Replaced on every push.
#
# The resolver binds its addresses once, at start (bind-interfaces), so it starts only once this
# machine's private address is there: a start that finds it missing fails after at most two seconds and
# is tried again every ten seconds (Restart=on-failure, no start limit) until the tunnel has brought it.
# It does not order itself After= or Wants= network-online.target, and its wait is short: the packaged
# unit is Before= network-online.target and nss-lookup.target, and the tunnel is ordered after them, so
# a long wait would hold the tunnel back and a drop-in ordering would be a cycle (issue 1126).
# machine's private address is there: it waits up to 80 seconds for it, and a start that found it
# missing is tried again every ten seconds rather than left failed.
[Unit]
After=network-online.target
Wants=network-online.target
StartLimitIntervalSec=0
[Service]
ExecStartPre=/usr/bin/sh -c 'for i in $(seq 1 2); do ip -4 -o addr show to ${machine:address} | grep -q . && exit 0; sleep 1; done; echo "${machine:address} is not on this machine" >&2; exit 1'
ExecStartPre=/usr/bin/sh -c 'for i in $(seq 1 80); do ip -4 -o addr show to ${machine:address} | grep -q . && exit 0; sleep 1; done; echo "${machine:address} is not on this machine" >&2; exit 1'
Restart=on-failure
RestartSec=10s
@@ -9,10 +9,7 @@
{
"name": "wildcard-resolution",
"scope": "mesh",
"identity": false,
"credential": {
"form": "none"
}
"identity": false
}
],
"requires": [
@@ -7,7 +7,7 @@
"privileged"
],
"invokes": [
"seat:mesh-artifact-sweep.images"
"seat:mesh-controller.images"
],
"claims": [
{
@@ -40,7 +40,6 @@
],
"emits": [
"repo.created",
"repo.writable",
"issue.opened",
"pull.merged",
"pull.updated",
@@ -182,12 +181,6 @@
"trusted": true,
"default": {}
},
"check-gates-merge": {
"type": "boolean",
"description": "Whether gitea_merge_pull_request judges mesh/check before it merges. True (the default): a head merges only on a passing mesh/check judged against the trunk's head (ADR 0447, ADR 0534 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"): a head the operator's Approve names in approved_heads merges whatever its mesh/check says (pending, failed, missing or judged on an older trunk); a head that moved, a branch no rule guards and a merge git refuses are still refused. Set together with mesh-delivery-gate's check-gates-merge.",
"trusted": true,
"default": true
},
"main-mergers": {
"type": "list",
"description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 §2): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.",
@@ -266,8 +259,7 @@
"env": {
"DB_TYPE": "postgres",
"USER_UID": "1000",
"USER_GID": "1000",
"SSH_MAX_STARTUPS": "100:30:200"
"USER_GID": "1000"
},
"env-file": [
"${dir:state}/server.env"
@@ -324,8 +316,7 @@
"takes-config": [
"url",
"keep-main-rules",
"main-mergers",
"check-gates-merge"
"main-mergers"
]
},
{
@@ -400,7 +391,6 @@
"provides": [
{
"name": "npm-package-registry",
"credential": {"form": "pair"},
"scope": "mesh",
"identity": {
"max": 40,
@@ -409,7 +399,6 @@
},
{
"name": "git",
"credential": {"form": "none"},
"scope": "mesh"
},
{
@@ -422,7 +411,6 @@
},
{
"name": "forge-account",
"credential": {"form": "none"},
"scope": "mesh"
}
],
@@ -549,7 +537,6 @@
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
"MESH_GITEA_ORIGINS_FILE": "${dir:mesh-state}/origins",
"MESH_GITEA_SSH_PORT": "222",
"MESH_GITEA_HOST_KEY_FILE": "${dir:data}/ssh/ssh_host_ed25519_key.pub",
"MESH_FORGE_HOST_FILE": "/etc/gitea/forge-host"
}
}
@@ -567,14 +554,11 @@
"announced"
],
"reads": [
"mesh-provisioner.plans",
"mesh-provisioner.provisions",
"mesh-controller.assignments",
"ssh-client.keys",
"mesh-build-go.keys",
"mesh-build-node.keys",
"mesh-build-image.keys",
"mesh-delivery-checker.owed",
"mesh-delivery-indexer.modules",
"mesh-delivery-gate.source"
"mesh-delivery-checker.owed"
]
}
@@ -7,10 +7,7 @@
"provides": [
{
"name": "go-toolchain",
"reach": "machine",
"credential": {
"form": "none"
}
"reach": "machine"
}
],
"resources": [
@@ -7,9 +7,6 @@
"scope": "mesh",
"identity": {
"in": "an InfluxDB v1 authorization"
},
"credential": {
"form": "pair"
}
}
],
@@ -1,4 +0,0 @@
# Written by the mesh's kernel module (systemd-modules-load reads it at boot).
# tun: buildah's network for a RUN line (pasta) and every other user-space tunnel open /dev/net/tun.
# Loaded at boot, so an upgrade that removes the running kernel's modules cannot keep it out later.
tun
@@ -100,25 +100,6 @@
"id": "linux-lts-headers",
"type": "package",
"package": "linux-lts-headers"
},
{
"id": "modules-load",
"type": "file",
"path": "/etc/modules-load.d/mesh-kernel.conf",
"mode": "0644",
"template": "files/modules-load.conf"
},
{
"id": "modules-keep",
"type": "package",
"package": "kernel-modules-hook"
},
{
"id": "modules-cleanup",
"type": "service",
"unit": "linux-modules-cleanup.service",
"state": "stopped",
"boot": "enabled"
}
],
"build": {
@@ -9,7 +9,6 @@
"provides": [
{
"name": "oidc-client",
"credential": {"form": "pair"},
"scope": "mesh",
"identity": {
"max": 255,
@@ -241,7 +240,6 @@
"bound"
],
"reads": [
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
"mesh-controller.assignments"
]
}
@@ -665,14 +665,7 @@
"path": "${dir:mesh-state}/config.json",
"mode": "0600",
"template": "files/config.json",
"takes-config": [
"domain",
"proxy-address",
"sitename",
"website",
"url",
"container"
],
"takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"],
"merge": "json"
},
{
@@ -765,9 +758,6 @@
"identity": {
"max": 64,
"in": "a mailbox's local part"
},
"credential": {
"form": "pair"
}
}
],
@@ -1,5 +1,3 @@
{
"capacity": 0,
"cores-per-step": 6,
"build-hours": ""
"capacity": "shanks=3,ace=2,g14=0,novox=0"
}
@@ -10,7 +10,7 @@
"state": [
"toolchains",
"keys",
{"name": "steps", "ttl-seconds": 172800}, "pauses"
"steps"
],
"claims": [
{
@@ -20,27 +20,14 @@
}
],
"reads": [
"mesh-delivery-checker.kept",
"mesh-delivery-indexer.manifests"
"mesh-delivery-checker.kept"
],
"config": {
"capacity": {
"type": "number",
"description": "How many asks of the Go build seat this module runs at once on this node (novox/hq ADR 0525 §5); 0 pulls nothing. It is set per node in the mesh's configuration; a node with no value builds nothing.",
"minimum": 0,
"default": 0
},
"build-hours": {
"type": "string",
"description": "When this node may take this module's build asks: one window HH:MM-HH:MM in the node's local time (an end before its start crosses midnight, 18:00-06:00). Outside it the node takes no new step and finishes the ones it runs; an empty value builds at any hour. It is set per node in the mesh's configuration.",
"pattern": "|([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9]",
"default": ""
},
"cores-per-step": {
"type": "number",
"description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.",
"minimum": 0.1,
"default": 6
"description": "How many asks of the Go build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
"default": "shanks=3,ace=2,g14=0,novox=0"
}
},
"tools": [
@@ -51,7 +38,6 @@
"build_go_release",
"build_go_step",
"build_go_steps",
"build_go_step_end",
"build_go_toolchains",
"build_go_status",
"build_go_health"
@@ -119,7 +105,7 @@
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "mesh-build-go",
"takes-config": ["capacity", "cores-per-step", "build-hours"],
"takes-config": ["capacity"],
"merge": "json",
"template": "files/settings.json"
},
@@ -1,5 +1,3 @@
{
"capacity": 0,
"cores-per-step": 2,
"build-hours": ""
"capacity": "shanks=3,ace=2,g14=0,novox=0"
}
@@ -9,7 +9,7 @@
],
"state": [
"keys",
{"name": "steps", "ttl-seconds": 172800}, "pauses"
"steps"
],
"claims": [
{
@@ -20,22 +20,10 @@
],
"config": {
"capacity": {
"type": "number",
"description": "How many asks of the image build seat this module runs at once on this node (novox/hq ADR 0525 §5); 0 pulls nothing. It is set per node in the mesh's configuration; a node with no value builds nothing.",
"minimum": 0,
"default": 0
},
"build-hours": {
"type": "string",
"description": "When this node may take this module's build asks: one window HH:MM-HH:MM in the node's local time (an end before its start crosses midnight, 18:00-06:00). Outside it the node takes no new step and finishes the ones it runs; an empty value builds at any hour. It is set per node in the mesh's configuration.",
"pattern": "|([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9]",
"default": ""
},
"cores-per-step": {
"type": "number",
"description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.",
"minimum": 0.1,
"default": 2
"description": "How many asks of the image build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
"default": "shanks=3,ace=2,g14=0,novox=0"
}
},
"tools": [
@@ -45,7 +33,6 @@
"build_image_release",
"build_image_step",
"build_image_steps",
"build_image_step_end",
"build_image_status",
"build_image_health"
],
@@ -90,7 +77,7 @@
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "mesh-build-image",
"takes-config": ["capacity", "cores-per-step", "build-hours"],
"takes-config": ["capacity"],
"merge": "json",
"template": "files/settings.json"
},
@@ -1,5 +1,3 @@
{
"capacity": 0,
"cores-per-step": 2,
"build-hours": ""
"capacity": "shanks=3,ace=2,g14=0,novox=0"
}
@@ -10,7 +10,7 @@
"state": [
"toolchains",
"keys",
{"name": "steps", "ttl-seconds": 172800}, "pauses"
"steps"
],
"claims": [
{
@@ -21,22 +21,10 @@
],
"config": {
"capacity": {
"type": "number",
"description": "How many asks of the Node build seat this module runs at once on this node (novox/hq ADR 0525 §5); 0 pulls nothing. It is set per node in the mesh's configuration; a node with no value builds nothing.",
"minimum": 0,
"default": 0
},
"build-hours": {
"type": "string",
"description": "When this node may take this module's build asks: one window HH:MM-HH:MM in the node's local time (an end before its start crosses midnight, 18:00-06:00). Outside it the node takes no new step and finishes the ones it runs; an empty value builds at any hour. It is set per node in the mesh's configuration.",
"pattern": "|([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9]",
"default": ""
},
"cores-per-step": {
"type": "number",
"description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.",
"minimum": 0.1,
"default": 2
"description": "How many asks of the Node build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
"default": "shanks=3,ace=2,g14=0,novox=0"
}
},
"tools": [
@@ -46,7 +34,6 @@
"build_node_release",
"build_node_step",
"build_node_steps",
"build_node_step_end",
"build_node_toolchains",
"build_node_status",
"build_node_health"
@@ -112,7 +99,7 @@
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "mesh-build-node",
"takes-config": ["capacity", "cores-per-step", "build-hours"],
"takes-config": ["capacity"],
"merge": "json",
"template": "files/settings.json"
},
@@ -25,7 +25,9 @@
"secrets": {
"postgres-database": "${dir:state}/database.secret"
},
"consumes": [],
"consumes": [
"mesh-controller.built-before"
],
"emits": [
"registered",
"upgraded",
@@ -90,12 +92,5 @@
"from": "files"
}
]
},
"invokes": [
"mesh-delivery-builder.builder_builds_since"
],
"reads": [
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.manifests"
]
}
}
@@ -1,87 +0,0 @@
{
"module": "mesh-conditions",
"version": "1",
"runs-as": "mesh-conditions",
"claims": [
{
"name": "mesh-conditions",
"scope": "mesh",
"serves": [
"conditions",
"silence",
"compare",
"take-over",
"hand-back",
"resync"
]
}
],
"uses": [
"operator-channel"
],
"reads": [
"mesh-delivery-watchman.findings",
"mesh-self-check.findings",
"mesh-healers.findings",
"mesh-delivery-assigner.findings",
"messenger.findings",
"mesh-provisioner.findings",
"artifact-store-tools.findings",
"mesh-healers.attempts",
"mesh-controller.conditions",
"mesh-controller.lease",
"mesh-delivery-assigner.wants",
"node-engine.have",
"mesh-delivery-assigner.placement"
],
"state": [
"conditions",
{
"name": "history",
"ttl-seconds": 7776000
},
"takeover",
"held",
{
"name": "bounds",
"key-ttl": true
},
"asked"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-conditions",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-conditions",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-conditions"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-conditions",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-conditions",
"binary": "mesh-conditions",
"loads": [
"mesh-conditions"
]
}
]
}
}
@@ -7,10 +7,7 @@
"name": "mesh-config",
"scope": "mesh",
"serves": [
"config",
"compare",
"take-over",
"hand-back"
"config"
]
}
],
@@ -26,30 +23,14 @@
"gitea.forge.pull-request.closed"
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.warranted",
"seat:mesh-controller.hand-act",
"seat:mesh-controller.warranted",
"gitea.gitea_get_pull_request"
],
"reads": [
"mesh-delivery-assigner.placement",
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests",
"mesh-controller.predefined",
"mesh-controller.assignments",
"mesh-controller.declared-config",
"mesh-delivery-indexer.predefined",
"mesh-delivery-indexer.config-candidates",
"mesh-controller.lease"
],
"tools": [
"config_compare",
"config_take_over",
"config_hand_back",
"config_declared",
"config_resync"
"mesh-controller.predefined"
],
"state": [
"values",
@@ -57,9 +38,7 @@
"predefined",
"proposals",
"pulls",
"kept",
"declared",
"takeover"
"kept"
],
"logs": [
"history"
@@ -1,5 +1,4 @@
{
"refusal-stall-floor-minutes": 10,
"author-new-head-minutes": 120,
"check-gates-merge": true
"author-new-head-minutes": 120
}
@@ -14,8 +14,7 @@
"table",
"stalled",
"recheck",
"close",
"settle"
"close"
]
}
],
@@ -27,17 +26,13 @@
"gitea.forge.pull-request.edited",
"gitea.forge.pull-request.closed",
"mesh-delivery-forge-bridge.forge.**",
"mesh-config.config-changed",
"gitea.repo.writable"
"mesh-config.config-changed"
],
"emits": [
"transition",
"group",
"merged"
],
"uses": [
"operator-channel"
],
"invokes": [
"gitea.gitea_commit_statuses",
"gitea.gitea_contains",
@@ -62,23 +57,14 @@
{
"name": "source",
"history": 64
},
"conditions",
{
"name": "turns",
"history": 5
}
],
"reads": [
"mesh-delivery-teller.answers",
"mesh-delivery-checker.verdicts",
"mesh-delivery-view.reach"
"mesh-delivery-checker.verdicts"
],
"tools": [
"delivery_hand_now",
"delivery_merge_again",
"delivery_status",
"delivery_turns"
"delivery_status"
],
"config": {
"refusal-stall-floor-minutes": {
@@ -96,12 +82,6 @@
"maximum": 10080,
"trusted": false,
"default": 120
},
"check-gates-merge": {
"type": "boolean",
"description": "Whether mesh/check gates the merge. True (the default): the operator's Approve is asked, and a merge handed, only on a passing verdict judged against the trunk's head, and a trunk's move puts a verdict out of date (ADR 0447, ADR 0505 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"; developers run the unit and integration levels locally before a pull request): a pull request reviewed at its head is ready, and its Approve is asked once the check at its heads (a group: of all its heads together) has a verdict, a pass or a failure said in the ask, or after 60 minutes with none, said so (the operator, 2026-10-27 23:54: \"if you don't want me to approve it, don't send the approval request\"); an approved merge waits for no verdict; no merge is refused for a check judged on an older trunk; a later verdict is shown on the pull request and the delivery, and asks a pending Approve again in its words; a group still merges all its members in one pass. Set it false together with the gitea module's check-gates-merge, which lets its merge tool merge an approved head without a passing mesh/check.",
"trusted": false,
"default": true
}
},
"resources": [
@@ -119,8 +99,7 @@
"mode": "0600",
"takes-config": [
"refusal-stall-floor-minutes",
"author-new-head-minutes",
"check-gates-merge"
"author-new-head-minutes"
],
"merge": "json",
"template": "files/settings.json"
@@ -1,65 +0,0 @@
{
"module": "mesh-hand-acts",
"version": "1",
"runs-as": "mesh-hand-acts",
"claims": [
{
"name": "mesh-hand-acts",
"scope": "mesh",
"serves": [
"hand-act",
"hand-acts",
"warranted"
]
}
],
"reads": [
"mesh-controller.hand-acts",
"messenger.asks"
],
"logs": [
{
"name": "acts",
"max-age-days": 400
}
],
"state": [
"warrants"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-hand-acts",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-hand-acts",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-hand-acts"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-hand-acts",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-hand-acts",
"binary": "mesh-hand-acts",
"loads": [
"mesh-hand-acts"
]
}
]
}
}
@@ -1,82 +0,0 @@
{
"module": "mesh-healers",
"version": "1",
"runs-as": "mesh-healers",
"claims": [
{
"name": "mesh-healers",
"scope": "mesh",
"serves": [
"healers",
"heal",
"drill"
]
}
],
"consumes": [
"mesh-conditions.condition-raised",
"mesh-conditions.condition-changed",
"mesh-hand-acts.act-recorded"
],
"invokes": [
"mesh-grants.grants_assert",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-controller.drill"
],
"reads": [
"mesh-controller.lease",
"mesh-conditions.conditions",
"mesh-controller.hand-acts"
],
"state": [
{
"name": "attempts",
"key-ttl": true
},
{
"name": "would",
"key-ttl": true
},
"findings",
{
"name": "wakes",
"key-ttl": true
}
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-healers",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-healers",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-healers"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-healers",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-healers",
"binary": "mesh-healers",
"loads": [
"mesh-healers"
]
}
]
}
}
@@ -62,13 +62,10 @@
"mesh-catalog modules/mesh-build-node",
"mesh-catalog modules/mesh-catalog",
"mesh-catalog modules/mesh-cli",
"mesh-catalog modules/mesh-conditions",
"mesh-catalog modules/mesh-config",
"mesh-catalog modules/mesh-delivery",
"mesh-catalog modules/mesh-delivery-gate",
"mesh-catalog modules/mesh-files",
"mesh-catalog modules/mesh-hand-acts",
"mesh-catalog modules/mesh-healers",
"mesh-catalog modules/mesh-issues",
"mesh-catalog modules/mesh-knowledge",
"mesh-catalog modules/mesh-mail",
@@ -78,7 +75,6 @@
"mesh-catalog modules/mesh-provisioner",
"mesh-catalog modules/mesh-provisioner-client",
"mesh-catalog modules/mesh-reads",
"mesh-catalog modules/mesh-self-check",
"mesh-catalog modules/mesh-vault",
"mesh-catalog modules/mesh-watcher",
"mesh-catalog modules/messenger",
@@ -13,17 +13,12 @@
"move",
"note",
"link",
"milestones",
"milestone-set",
"milestone-move",
"import",
"groom",
"set",
"tracking",
"reserve",
"fix-times",
"learn-fixes",
"changes"
"fix-times"
]
}
],
@@ -38,17 +33,12 @@
"move",
"note",
"link",
"milestones",
"milestone-set",
"milestone-move",
"import",
"groom",
"set",
"tracking",
"reserve",
"fix-times",
"learn-fixes",
"changes"
"fix-times"
]
}
],
@@ -62,9 +52,7 @@
"triaged",
"ready",
"ready-to-verify",
"reserved",
"milestone-set",
"milestone-moved"
"reserved"
],
"consumes": [
"mesh-issues.opened",
@@ -78,7 +66,6 @@
"seat:operator-channel.notify",
"gitea.gitea_pull_reviews",
"gitea.gitea_contains",
"gitea.gitea_get_pull_request",
"records.records_status",
"records.records_list",
"records.records_read",
@@ -94,11 +81,7 @@
},
"counters",
"grooming",
"reservations",
{
"name": "milestones",
"history": 5
}
"reservations"
],
"config": {
"first-number": {
@@ -174,13 +157,10 @@
"mesh-catalog modules/mesh-build-node",
"mesh-catalog modules/mesh-catalog",
"mesh-catalog modules/mesh-cli",
"mesh-catalog modules/mesh-conditions",
"mesh-catalog modules/mesh-config",
"mesh-catalog modules/mesh-delivery",
"mesh-catalog modules/mesh-delivery-gate",
"mesh-catalog modules/mesh-files",
"mesh-catalog modules/mesh-hand-acts",
"mesh-catalog modules/mesh-healers",
"mesh-catalog modules/mesh-issues",
"mesh-catalog modules/mesh-knowledge",
"mesh-catalog modules/mesh-mail",
@@ -190,7 +170,6 @@
"mesh-catalog modules/mesh-provisioner",
"mesh-catalog modules/mesh-provisioner-client",
"mesh-catalog modules/mesh-reads",
"mesh-catalog modules/mesh-self-check",
"mesh-catalog modules/mesh-vault",
"mesh-catalog modules/mesh-watcher",
"mesh-catalog modules/messenger",
@@ -32,8 +32,7 @@
"propose",
"import",
"return",
"withdraw",
"progress"
"withdraw"
]
}
],
@@ -53,15 +52,13 @@
"propose",
"import",
"return",
"withdraw",
"progress"
"withdraw"
]
}
],
"tools": [
"knowledge_health",
"knowledge_highest",
"knowledge_marks"
"knowledge_highest"
],
"emits": [
"reserved",
@@ -84,14 +81,7 @@
"records.records_status",
"records.records_list",
"records.records_read",
"seat:operator-channel.notify",
"seat:issue-tracker.list",
"seat:issue-tracker.get",
"seat:issue-tracker.milestones",
"seat:issue-tracker.changes",
"seat:mesh-delivery.deliveries",
"seat:mesh-delivery.show",
"seat:mesh-delivery.groups"
"seat:operator-channel.notify"
],
"config": {
"judge-model": {
@@ -11,36 +11,11 @@
]
}
],
"uses": [
"operator-channel"
],
"reads": [
"mesh-delivery-assigner.placement",
"mesh-delivery-assigner.wants",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests",
"mesh-delivery-builder.builds",
"node-engine.have",
"mesh-config.resolved",
"gitea.bound",
"keycloak.bound",
"postgres.bound",
"route-adapter.bound",
"mesh-vault.bound"
],
"tools": [
"provisioner_provisions",
"provisioner_plans",
"provisioner_compose",
"provisioner_decide"
],
"state": [
"registry",
"needs",
"applied",
"collect",
"provisions",
"plans"
"collect"
],
"own-secrets": {
"broker": "${dir:state}/broker"
@@ -1,87 +0,0 @@
{
"module": "mesh-self-check",
"version": "1",
"runs-as": "mesh-self-check",
"claims": [
{
"name": "mesh-self-check",
"scope": "mesh",
"serves": [
"doctor",
"probes"
]
}
],
"invokes": [
"nats.nats_streams",
"nats.nats_closed_connections",
"seat:mesh-watchdogs.signals",
"seat:node-intrusion-prevention.banned"
],
"reads": [
"mesh-delivery-assigner.placement",
"mesh-delivery-assigner.seat-holders",
"mesh-delivery-watchman.presence",
"mesh-host.health",
"disk-load.readings",
"restic.good",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests"
],
"state": [
"findings",
"verdict",
{
"name": "verdicts",
"ttl-seconds": 604800
}
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-self-check",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-self-check",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-self-check"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-self-check",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-self-check",
"binary": "mesh-self-check",
"loads": [
"mesh-self-check"
]
}
]
},
"config": {
"probes": {
"type": "list",
"description": "The ids of the probes that run, each judging one thing of the mesh (mesh-self-check.probes lists them with what each asserts). A probe not listed runs nothing and raises nothing, and what it raised before is cleared. D8 and DA are in the code and off by default.",
"trusted": true,
"default": [
"D13",
"D-fill"
],
"runtime": true
}
}
}
@@ -5,9 +5,6 @@
"provides": [
{
"name": "secret",
"credential": {
"form": "pair"
},
"scope": "mesh",
"identity": {
"in": "a vault entry"
@@ -17,27 +14,10 @@
"emits": [
"provisioned",
"rotated",
"secret-changed",
"account-hashed",
"own-replaced",
"account-issued",
"signed"
"secret-changed"
],
"reads": [
"node-engine.have",
"mesh-grants.accounts",
"mesh-controller.lease",
"mesh-delivery-assigner.placement",
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
],
"state": [
{
"name": "bus-accounts",
"history": 2
},
"signers",
"bound"
"mesh-controller.assignments"
],
"receives": {
"secret": "${dir:grants}/mesh.json"
@@ -155,8 +135,7 @@
"MESH_VAULT_LEDGER": "${dir:ledger}",
"MESH_VAULT_ROOT": "${dir:root}",
"MESH_VAULT_STORE": "${dir:store}",
"MESH_VAULT_STATE": "${dir:state}",
"MESH_VAULT_SECRET_ASK_CALLERS": "node-tools"
"MESH_VAULT_STATE": "${dir:state}"
}
}
]
@@ -166,10 +145,6 @@
"secret_holders",
"secret_verify",
"secret_handed",
"secret_own",
"account_hash",
"account_adopt",
"account_retire",
"secret_asks",
"secret_vault_export",
"provisioner_retirement",
@@ -177,12 +152,7 @@
"provisioner_retire_reject",
"provisioner_delete",
"vault_status",
"vault_health",
"own_drift",
"own_take_switch",
"vault_signers",
"vault_signed",
"signers_reassert"
"vault_health"
],
"claims": [
{
@@ -196,10 +166,7 @@
"rotate",
"move-in",
"node-key",
"move-asks",
"own",
"issue-account",
"sign"
"move-asks"
]
}
]

Some files were not shown because too many files have changed in this diff Show More